diff --git a/.travis.yml b/.travis.yml index 6815c806..c5eb2a98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ install: - go get github.com/gorilla/mux - go get github.com/mattn/go-sqlite3 - go get github.com/jinzhu/gorm +- go get github.com/Sirupsen/logrus +- go get gopkg.in/natefinch/lumberjack.v2 - go build deploy: provider: releases diff --git a/README.md b/README.md index aeeed88f..828247fd 100644 --- a/README.md +++ b/README.md @@ -34,13 +34,14 @@ The provided unit file uses options directly; if you prefer a config file, do th * Replace in the unit file the options by `-conf /etc/nyaa.conf` ## TODO -* RSS feeds(work in progress) -* torrent sorting (work in progress) +* adding new magnet links * API improvement +* add rss link and generate link based on your current search +* make renchon the favicon and official mascot +* make sukebei db schema compatible with current code +* comments in torrent view page * Site theme -* Torrent view and description page(work in progress) * accounts? -* Adding new torrents * scraping * Daily DB dumps * p2p sync of dbs? diff --git a/config.go b/config/config.go similarity index 91% rename from config.go rename to config/config.go index 33b8f418..addac6ef 100644 --- a/config.go +++ b/config/config.go @@ -1,4 +1,4 @@ -package main +package config import ( "bufio" @@ -50,7 +50,7 @@ func (config *Config) BindFlags() processFlags { db_params := flag.String("dbparams", Defaults.DBParams, "parameters to open the database (see Gorm's doc)") return func() error { - err := config.handleConfFileFlag(*conf_file) + err := config.HandleConfFileFlag(*conf_file) if err != nil { return err } @@ -58,12 +58,12 @@ func (config *Config) BindFlags() processFlags { config.Host = *host config.Port = *port config.DBParams = *db_params - err = config.setDBType(*db_type) + err = config.SetDBType(*db_type) return err } } -func (config *Config) handleConfFileFlag(path string) error { +func (config *Config) HandleConfFileFlag(path string) error { if path != "" { file, err := os.Open(path) if err != nil { @@ -78,7 +78,7 @@ func (config *Config) handleConfFileFlag(path string) error { return nil } -func (config *Config) setDBType(db_type string) error { +func (config *Config) SetDBType(db_type string) error { if !allowedDatabaseTypes[db_type] { return errors.New(fmt.Sprintf("Unknown database backend '%s'.", db_type)) } @@ -96,10 +96,10 @@ func (config *Config) Write(output io.Writer) error { func (config *Config) Pretty(output io.Writer) error { data, err := json.MarshalIndent(config, "", "\t") - data = append(data, []byte("\n")...) if err != nil { return err } + data = append(data, []byte("\n")...) _, err = output.Write(data) return err } diff --git a/config/env.go b/config/env.go new file mode 100644 index 00000000..3999a39d --- /dev/null +++ b/config/env.go @@ -0,0 +1,8 @@ +package config + +// Constants for environment. +const ( + // DEVELOPMENT | TEST | PRODUCTION + Environment = "DEVELOPMENT" + // Environment = "PRODUCTION" +) diff --git a/config/logger.go b/config/logger.go new file mode 100644 index 00000000..dcdccdbd --- /dev/null +++ b/config/logger.go @@ -0,0 +1,15 @@ +package config + +// Constants for logger. +const ( + AccessLogFilePath = "log/access" + AccessLogFileExtension = ".txt" + AccessLogMaxSize = 5 // megabytes + AccessLogMaxBackups = 7 + AccessLogMaxAge = 30 //days + ErrorLogFilePath = "log/error" + ErrorLogFileExtension = ".json" + ErrorLogMaxSize = 10 // megabytes + ErrorLogMaxBackups = 7 + ErrorLogMaxAge = 30 //days +) diff --git a/config/navigation.go b/config/navigation.go new file mode 100644 index 00000000..ade0972d --- /dev/null +++ b/config/navigation.go @@ -0,0 +1,6 @@ +package config + +// Constants for pagination. +const ( + TorrentsPerPage = 50 +) diff --git a/config/sorting.go b/config/sorting.go new file mode 100644 index 00000000..41df588a --- /dev/null +++ b/config/sorting.go @@ -0,0 +1,7 @@ +package config + +// Constants for ordering models. +const ( + TorrentOrder = "torrent_id" + TorrentSort = "DESC" +) diff --git a/config/trackers.go b/config/trackers.go new file mode 100644 index 00000000..ed0eed33 --- /dev/null +++ b/config/trackers.go @@ -0,0 +1,5 @@ +package config + +const ( + Trackers = "&tr=udp://zer0day.to:1337/announce&tr=udp://tracker.leechers-paradise.org:6969&tr=udp://explodie.org:6969&tr=udp://tracker.opentrackr.org:1337&tr=udp://tracker.coppersurfer.tk:6969&tr=http://tracker.baka-sub.cf/announce" +) \ No newline at end of file diff --git a/css/style.css b/css/style.css index 406f043a..5dd91a5c 100644 --- a/css/style.css +++ b/css/style.css @@ -16,7 +16,7 @@ nav#mainmenu { position: fixed; color: white; width: 100%; - z-index: 3; + z-index: 4; } nav#mainmenu a { @@ -72,6 +72,11 @@ a { white-space: nowrap; } +.table > tbody > tr > td { + /* fix bootstrap uglyness */ + vertical-align: middle; +} + div.container div.blockBody:nth-of-type(2) table{table-layout:fixed;} div.container div.blockBody:nth-of-type(2) table tr:first-of-type th:first-of-type, div.container div.blockBody:nth-of-type(2) table tr:first-of-type th:nth-of-type(5){width:10%;} div.container div.blockBody:nth-of-type(2) table tr:first-of-type th:nth-of-type(3){width:15%;} diff --git a/db/gorm.go b/db/gorm.go new file mode 100644 index 00000000..42ba2bf9 --- /dev/null +++ b/db/gorm.go @@ -0,0 +1,58 @@ +package db + +import ( + "github.com/ewhal/nyaa/config" + "github.com/ewhal/nyaa/util/log" + "github.com/ewhal/nyaa/model" + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/sqlite" + // _ "github.com/go-sql-driver/mysql" +) + +var ORM, Errs = GormInit() + +// GormInit init gorm ORM. +func GormInit() (*gorm.DB, error) { + conf := config.NewConfig() + db, err := gorm.Open(conf.DBType, conf.DBParams) + // db, err := gorm.Open("mysql", config.MysqlDSL()) + //db, err := gorm.Open("sqlite3", "/tmp/gorm.db") + + // Get database connection handle [*sql.DB](http://golang.org/pkg/database/sql/#DB) + db.DB() + + // Then you could invoke `*sql.DB`'s functions with it + db.DB().Ping() + db.DB().SetMaxIdleConns(10) + db.DB().SetMaxOpenConns(100) + + // Disable table name's pluralization + // db.SingularTable(true) + if config.Environment == "DEVELOPMENT" { + db.LogMode(true) + // db.DropTable(&model.User{}, "UserFollower") + db.AutoMigrate(&model.Torrents{}, &model.Categories{}, &model.Sub_Categories{}, &model.Statuses{}) + // db.AutoMigrate(&model.User{}, &model.Role{}, &model.Connection{}, &model.Language{}, &model.Article{}, &model.Location{}, &model.Comment{}, &model.File{}) + // db.Model(&model.User{}).AddIndex("idx_user_token", "token") + + } + log.CheckError(err) + + // relation := gorm.Relationship{} + // relation.Kind = "many2many" + // relation.ForeignFieldNames = []string{"id"} //(M1 pkey) + // relation.ForeignDBNames = []string{"user_id"} //(M1 fkey in m1m2join) + // relation.AssociationForeignFieldNames = []string{"id"} //(M2 pkey) + // // relation.AssociationForeignStructFieldNames = []string{"id", "ID"} //(m2 pkey name in m2 struct?) + // relation.AssociationForeignDBNames = []string{"follower_id"} //(m2 fkey in m1m2join) + // m1Type := reflect.TypeOf(model.User{}) + // m2Type := reflect.TypeOf(model.User{}) + // handler := gorm.JoinTableHandler{} + // // ORDER BELOW MATTERS + // // Install handler + // db.SetJoinTableHandler(&model.User{}, "Likings", &handler) + // // Configure handler to use the relation that we've defined + // handler.Setup(&relation, "users_followers", m1Type, m2Type) + + return db, err +} diff --git a/js/main.js b/js/main.js index 04089014..2e9e87fa 100644 --- a/js/main.js +++ b/js/main.js @@ -22,12 +22,18 @@ var maxId = 5; for (var i = 0; i < maxId; i++) { var el = document.getElementById('page-' + i), n = prev + i; + if (el == null) + continue; el.href = pageString + n + query; el.innerHTML = n; } - document.getElementById('page-next').href = pageString + next + query; - document.getElementById('page-prev').href = pageString + prev + query; + var e = document.getElementById('page-next'); + if (e != null) + e.href = pageString + next + query; + var e = document.getElementById('page-prev'); + if (e != null) + e.href = pageString + prev + query; // Used by spoiler tags function toggleLayer(elem) { @@ -36,3 +42,33 @@ function toggleLayer(elem) { else elem.classList.add("hide"); } + +function formatDate(date) { // thanks stackoverflow + var monthNames = [ + "January", "February", "March", + "April", "May", "June", "July", + "August", "September", "October", + "November", "December" + ]; + + var day = date.getDate(); + var monthIndex = date.getMonth(); + var year = date.getFullYear(); + + return day + ' ' + monthNames[monthIndex] + ' ' + year; +} + +var list = document.getElementsByClassName("date-short"); +for(var i in list) { + var e = list[i]; + e.title = e.innerText; + e.innerText = formatDate(new Date(e.innerText)); +} + +var list = document.getElementsByClassName("date-full"); +for(var i in list) { + var e = list[i]; + e.title = e.innerText; + var date = new Date(e.innerText); + e.innerText = date.toDateString() + " " + date.toLocaleTimeString(); +} diff --git a/main.go b/main.go index b3c783c4..36aac784 100644 --- a/main.go +++ b/main.go @@ -1,20 +1,20 @@ package main import ( - "bufio" - "bytes" - "compress/zlib" +"bufio" "encoding/json" "fmt" "flag" "github.com/gorilla/feeds" "github.com/gorilla/mux" - "github.com/jinzhu/gorm" - _ "github.com/jinzhu/gorm/dialects/sqlite" + + "github.com/ewhal/nyaa/model" + "github.com/ewhal/nyaa/service/torrent" + "github.com/ewhal/nyaa/util/log" + "github.com/ewhal/nyaa/config" + "html" "html/template" - "io/ioutil" - "log" "net/http" "os" "strconv" @@ -22,6 +22,7 @@ import ( "time" ) +var router *mux.Router type SearchParam struct { Category string Order string @@ -31,54 +32,19 @@ type SearchParam struct { Sort string } -var db *gorm.DB -var router *mux.Router -var debugLogger *log.Logger -var trackers = "&tr=udp://tracker.coppersurfer.tk:6969&tr=udp://zer0day.to:1337/announce&tr=udp://tracker.leechers-paradise.org:6969&tr=udp://explodie.org:6969&tr=udp://tracker.opentrackr.org:1337&tr=http://tracker.baka-sub.cf/announce" - -func getDBHandle(db_type string, db_params string) *gorm.DB { - dbInit, err := gorm.Open(db_type, db_params) - - // Migrate the schema of Torrents - dbInit.AutoMigrate(&Torrents{}, &Categories{}, &Sub_Categories{}, &Statuses{}) - - checkErr(err) - return dbInit -} - -func checkErr(err error) { - if err != nil { - debugLogger.Println(" " + err.Error()) - } -} - -func unZlib(description []byte) string { - if len(description) > 0 { - b := bytes.NewReader(description) - //log.Println(b) - z, err := zlib.NewReader(b) - checkErr(err) - defer z.Close() - p, err := ioutil.ReadAll(z) - checkErr(err) - return string(p) - } - return "" -} - func apiHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) page := vars["page"] pagenum, _ := strconv.Atoi(html.EscapeString(page)) - b := CategoryJson{Torrents: []TorrentsJson{}} + b := model.CategoryJson{Torrents: []model.TorrentsJson{}} maxPerPage := 50 nbTorrents := 0 - torrents, nbTorrents := getAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) + torrents, nbTorrents := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) for i, _ := range torrents { - res := torrents[i].toJson() + res := torrents[i].ToJson() b.Torrents = append(b.Torrents, res) } @@ -97,10 +63,10 @@ func apiViewHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id := vars["id"] - b := CategoryJson{Torrents: []TorrentsJson{}} + b := model.CategoryJson{Torrents: []model.TorrentsJson{}} - torrent, err := getTorrentById(id) - res := torrent.toJson() + torrent, err := torrentService.GetTorrentById(id) + res := torrent.ToJson() b.Torrents = append(b.Torrents, res) b.QueryRecordCount = 1 @@ -116,7 +82,7 @@ func apiViewHandler(w http.ResponseWriter, r *http.Request) { func searchHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.New("home").Funcs(funcMap).ParseFiles("templates/index.html", "templates/home.html")) - templates.ParseGlob("templates/_*.html") // common + templates.ParseGlob("templates/_*.html") // common vars := mux.Vars(r) page := vars["page"] @@ -126,12 +92,12 @@ func searchHandler(w http.ResponseWriter, r *http.Request) { pagenum = 1 } - b := []TorrentsJson{} + b := []model.TorrentsJson{} search_param, torrents, nbTorrents := searchByQuery( r, pagenum ) for i, _ := range torrents { - res := torrents[i].toJson() + res := torrents[i].ToJson() b = append(b, res) } @@ -143,7 +109,7 @@ func searchHandler(w http.ResponseWriter, r *http.Request) { search_param.Sort, search_param.Order, } - htv := HomeTemplateVariables{b, getAllCategories(false), searchForm, navigationTorrents, r.URL, mux.CurrentRoute(r)} + htv := HomeTemplateVariables{b, torrentService.GetAllCategories(false), searchForm, navigationTorrents, r.URL, mux.CurrentRoute(r)} err := templates.ExecuteTemplate(w, "index.html", htv) if err != nil { @@ -151,7 +117,7 @@ func searchHandler(w http.ResponseWriter, r *http.Request) { } } -func searchByQuery(r *http.Request, pagenum int) (SearchParam, []Torrents, int) { +func searchByQuery(r *http.Request, pagenum int) (SearchParam, []model.Torrents, int) { maxPerPage, errConv := strconv.Atoi(r.URL.Query().Get("max")) if errConv != nil { maxPerPage = 50 // default Value maxPerPage @@ -169,6 +135,7 @@ func searchByQuery(r *http.Request, pagenum int) (SearchParam, []Torrents, int) // need this to prevent out of index panics var searchCatId, searchSubCatId string if len(catsSplit) == 2 { + searchCatId = html.EscapeString(catsSplit[0]) searchSubCatId = html.EscapeString(catsSplit[1]) } @@ -180,15 +147,15 @@ func searchByQuery(r *http.Request, pagenum int) (SearchParam, []Torrents, int) } order_by := search_param.Sort + " " + search_param.Order - parameters := WhereParams{} + parameters := torrentService.WhereParams{} conditions := []string{} if searchCatId != "" { conditions = append(conditions, "category_id = ?") - parameters.params = append(parameters.params, searchCatId) + parameters.Params = append(parameters.Params, searchCatId) } if searchSubCatId != "" { conditions = append(conditions, "sub_category_id = ?") - parameters.params = append(parameters.params, searchSubCatId) + parameters.Params = append(parameters.Params, searchSubCatId) } if search_param.Status != "" { if search_param.Status == "2" { @@ -201,22 +168,18 @@ func searchByQuery(r *http.Request, pagenum int) (SearchParam, []Torrents, int) searchQuerySplit := strings.Split(search_param.Query, " ") for i, _ := range searchQuerySplit { conditions = append(conditions, "torrent_name LIKE ?") - parameters.params = append(parameters.params, "%"+searchQuerySplit[i]+"%") + parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") } - parameters.conditions = strings.Join(conditions[:], " AND ") - log.Printf("SQL query is :: %s\n", parameters.conditions) - torrents, n := getTorrentsOrderBy(¶meters, order_by, maxPerPage, maxPerPage*(pagenum-1)) + parameters.Conditions = strings.Join(conditions[:], " AND ") + log.Infof("SQL query is :: %s\n", parameters.Conditions) + torrents, n := torrentService.GetTorrentsOrderBy(¶meters, order_by, maxPerPage, maxPerPage*(pagenum-1)) return search_param, torrents, n } -func safe(s string) template.URL { - return template.URL(s) -} - func faqHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.New("FAQ").Funcs(funcMap).ParseFiles("templates/index.html", "templates/FAQ.html")) - templates.ParseGlob("templates/_*.html") // common + templates.ParseGlob("templates/_*.html") // common err := templates.ExecuteTemplate(w, "index.html", FaqTemplateVariables{Navigation{}, NewSearchForm(), r.URL, mux.CurrentRoute(r)}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -224,8 +187,10 @@ func faqHandler(w http.ResponseWriter, r *http.Request) { } func rssHandler(w http.ResponseWriter, r *http.Request) { + _, torrents, _ := searchByQuery( r, 1 ) created_as_time := time.Now() + if len(torrents) > 0 { created_as_time = time.Unix(torrents[0].Date, 0) } @@ -239,7 +204,7 @@ func rssHandler(w http.ResponseWriter, r *http.Request) { for i, _ := range torrents { timestamp_as_time := time.Unix(torrents[0].Date, 0) - torrent_json := torrents[i].toJson() + torrent_json := torrents[i].ToJson() feed.Items[i] = &feeds.Item{ // need a torrent view first //Id: URL + torrents[i].Hash, @@ -261,12 +226,12 @@ func rssHandler(w http.ResponseWriter, r *http.Request) { func viewHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.ParseFiles("templates/index.html", "templates/view.html")) - templates.ParseGlob("templates/_*.html") // common + templates.ParseGlob("templates/_*.html") // common vars := mux.Vars(r) id := vars["id"] - torrent, err := getTorrentById(id) - b := torrent.toJson() + torrent, err := torrentService.GetTorrentById(id) + b := torrent.ToJson() htv := ViewTemplateVariables{b, NewSearchForm(), Navigation{}, r.URL, mux.CurrentRoute(r)} @@ -277,9 +242,9 @@ func viewHandler(w http.ResponseWriter, r *http.Request) { } func rootHandler(w http.ResponseWriter, r *http.Request) { - var templates = template.Must(template.New("home").Funcs(funcMap).ParseFiles("templates/index.html", "templates/home.html")) +var templates = template.Must(template.New("home").Funcs(funcMap).ParseFiles("templates/index.html", "templates/home.html")) templates.ParseGlob("templates/_*.html") // common - vars := mux.Vars(r) + vars := mux.Vars(r) page := vars["page"] // db params url @@ -294,16 +259,16 @@ func rootHandler(w http.ResponseWriter, r *http.Request) { pagenum = 1 } - b := []TorrentsJson{} - torrents, nbTorrents := getAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) + b := []model.TorrentsJson{} + torrents, nbTorrents := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) for i, _ := range torrents { - res := torrents[i].toJson() + res := torrents[i].ToJson() b = append(b, res) } navigationTorrents := Navigation{nbTorrents, maxPerPage, pagenum, "search_page"} - htv := HomeTemplateVariables{b, getAllCategories(false), NewSearchForm(), navigationTorrents, r.URL, mux.CurrentRoute(r)} + htv := HomeTemplateVariables{b, torrentService.GetAllCategories(false), NewSearchForm(), navigationTorrents, r.URL, mux.CurrentRoute(r)} err := templates.ExecuteTemplate(w, "index.html", htv) if err != nil { @@ -312,8 +277,7 @@ func rootHandler(w http.ResponseWriter, r *http.Request) { } -func RunServer(conf *Config) { - db = getDBHandle(conf.DBType, conf.DBParams) +func RunServer(conf *config.Config) { router = mux.NewRouter() cssHandler := http.FileServer(http.Dir("./css/")) @@ -344,12 +308,12 @@ func RunServer(conf *Config) { } err := srv.ListenAndServe() - checkErr(err) + log.CheckError(err) } func main() { - conf := NewConfig() + conf := config.NewConfig() conf_bind := conf.BindFlags() defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout") flag.Parse() diff --git a/models.go b/model/torrent.go similarity index 53% rename from models.go rename to model/torrent.go index 52cf0ba3..58f0aeb3 100644 --- a/models.go +++ b/model/torrent.go @@ -1,8 +1,9 @@ -package main +package model import ( - "errors" - "github.com/jinzhu/gorm" + "github.com/ewhal/nyaa/util" + "github.com/ewhal/nyaa/config" + "html" "html/template" "strconv" @@ -10,6 +11,14 @@ import ( "time" ) +type Feed struct { + Id int + Name string + Hash string + Magnet string + Timestamp string +} + type Categories struct { Id int `gorm:"column:category_id"` Name string `gorm:"column:category_name"` @@ -78,107 +87,11 @@ type TorrentsJson struct { Magnet template.URL `json: "magnet"` } -type WhereParams struct { - conditions string // Ex : name LIKE ? AND category_id LIKE ? - params []interface{} -} - -/* Function to interact with Models - * - * Get the torrents with where clause - * - */ - -func getTorrentById(id string) (Torrents, error) { - var torrent Torrents - - if db.Where("torrent_id = ?", id).Find(&torrent).RecordNotFound() { - return torrent, errors.New("Article is not found.") - } - - return torrent, nil -} - -func getTorrentsOrderBy(parameters *WhereParams, orderBy string, limit int, offset int) ([]Torrents, int) { - var torrents []Torrents - var dbQuery *gorm.DB - var count int - conditions := "torrent_hash is not null" //filter out broken entries - var params []interface{} - if parameters != nil { // if there is where parameters - conditions += " AND " + parameters.conditions - params = parameters.params - } - db.Model(&torrents).Where(conditions, params...).Count(&count) - dbQuery = db.Model(&torrents).Where(conditions, params...) - - if orderBy == "" { - orderBy = "torrent_id DESC" - } // Default OrderBy - if limit != 0 || offset != 0 { // if limits provided - dbQuery = dbQuery.Limit(limit).Offset(offset) - } - dbQuery.Order(orderBy).Preload("Categories").Preload("Sub_Categories").Find(&torrents) - return torrents, count - -} - -/* Functions to simplify the get parameters of the main function - * - * Get Torrents with where parameters and limits, order by default - */ -func getTorrents(parameters WhereParams, limit int, offset int) ([]Torrents, int) { - return getTorrentsOrderBy(¶meters, "", limit, offset) -} - -/* Get Torrents with where parameters but no limit and order by default (get all the torrents corresponding in the db) - */ -func getTorrentsDB(parameters WhereParams) ([]Torrents, int) { - return getTorrentsOrderBy(¶meters, "", 0, 0) -} - -/* Function to get all torrents - */ - -func getAllTorrentsOrderBy(orderBy string, limit int, offset int) ([]Torrents, int) { - - return getTorrentsOrderBy(nil, orderBy, limit, offset) -} - -func getAllTorrents(limit int, offset int) ([]Torrents, int) { - return getTorrentsOrderBy(nil, "", limit, offset) -} - -func getAllTorrentsDB() ([]Torrents, int) { - return getTorrentsOrderBy(nil, "", 0, 0) -} - -/* Function to get all categories with/without torrents (bool) - */ -func getAllCategories(populatedWithTorrents bool) []Categories { - var categories []Categories - if populatedWithTorrents { - db.Preload("Torrents").Preload("Sub_Categories").Find(&categories) - } else { - db.Preload("Sub_Categories").Find(&categories) - } - return categories -} - -func createWhereParams(conditions string, params ...string) WhereParams { - whereParams := WhereParams{} - whereParams.conditions = conditions - for i, _ := range params { - whereParams.params = append(whereParams.params, params[i]) - } - - return whereParams -} /* Model Conversion to Json */ -func (t *Torrents) toJson() TorrentsJson { - magnet := "magnet:?xt=urn:btih:" + strings.TrimSpace(t.Hash) + "&dn=" + t.Name + trackers +func (t *Torrents) ToJson() TorrentsJson { + magnet := "magnet:?xt=urn:btih:" + strings.TrimSpace(t.Hash) + "&dn=" + t.Name + config.Trackers res := TorrentsJson{ Id: strconv.Itoa(t.Id), Name: html.UnescapeString(t.Name), @@ -186,21 +99,21 @@ func (t *Torrents) toJson() TorrentsJson { Hash: t.Hash, Date: time.Unix(t.Date, 0).Format(time.RFC3339), Filesize: t.Filesize, - Description: template.HTML(unZlib(t.Description)), - Sub_Category: t.Sub_Categories.toJson(), - Category: t.Categories.toJson(), - Magnet: safe(magnet)} + Description: template.HTML(util.UnZlib(t.Description)), + Sub_Category: t.Sub_Categories.ToJson(), + Category: t.Categories.ToJson(), + Magnet: util.Safe(magnet)} return res } -func (c *Sub_Categories) toJson() SubCategoryJson { +func (c *Sub_Categories) ToJson() SubCategoryJson { return SubCategoryJson{ Id: strconv.Itoa(c.Id), Name: html.UnescapeString(c.Name)} } -func (c *Categories) toJson() CategoryJson { +func (c *Categories) ToJson() CategoryJson { return CategoryJson{ Id: strconv.Itoa(c.Id), Name: html.UnescapeString(c.Name)} diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go new file mode 100644 index 00000000..c057d9c6 --- /dev/null +++ b/service/torrent/torrent.go @@ -0,0 +1,127 @@ +package torrentService +import ( + "github.com/ewhal/nyaa/db" + "github.com/ewhal/nyaa/model" + "github.com/ewhal/nyaa/config" + "github.com/jinzhu/gorm" + "errors" + "strings" +) + +type WhereParams struct { + Conditions string // Ex : name LIKE ? AND category_id LIKE ? + Params []interface{} +} + +/* Function to interact with Models + * + * Get the torrents with where clause + * + */ + +// don't need raw SQL once we get MySQL +func GetFeeds() []model.Feed { + var result []model.Feed + rows, err := db.ORM.DB(). + Query( + "SELECT `torrent_id` AS `id`, `torrent_name` AS `name`, `torrent_hash` AS `hash`, `timestamp` FROM `torrents` " + + "ORDER BY `timestamp` desc LIMIT 50") + if err == nil { + for rows.Next() { + item := model.Feed{} + rows.Scan(&item.Id, &item.Name, &item.Hash, &item.Timestamp) + magnet := "magnet:?xt=urn:btih:" + strings.TrimSpace(item.Hash) + "&dn=" + item.Name + config.Trackers + item.Magnet = magnet + // memory hog + result = append(result, item) + } + rows.Close() + } + return result +} + +func GetTorrentById(id string) (model.Torrents, error) { + var torrent model.Torrents + + if db.ORM.Where("torrent_id = ?", id).Find(&torrent).RecordNotFound() { + return torrent, errors.New("Article is not found.") + } + + return torrent, nil +} + +func GetTorrentsOrderBy(parameters *WhereParams, orderBy string, limit int, offset int) ([]model.Torrents, int) { + var torrents []model.Torrents + var dbQuery *gorm.DB + var count int + conditions := "torrent_hash is not null" //filter out broken entries + var params []interface{} + if parameters != nil { // if there is where parameters + conditions += " AND " + parameters.Conditions + params = parameters.Params + } + db.ORM.Model(&torrents).Where(conditions, params...).Count(&count) + dbQuery = db.ORM.Model(&torrents).Where(conditions, params...) + + if orderBy == "" { + orderBy = "torrent_id DESC" + } // Default OrderBy + if limit != 0 || offset != 0 { // if limits provided + dbQuery = dbQuery.Limit(limit).Offset(offset) + } + dbQuery.Order(orderBy).Preload("Categories").Preload("Sub_Categories").Find(&torrents) + return torrents, count + +} + +/* Functions to simplify the get parameters of the main function + * + * Get Torrents with where parameters and limits, order by default + */ +func GetTorrents(parameters WhereParams, limit int, offset int) ([]model.Torrents, int) { + return GetTorrentsOrderBy(¶meters, "", limit, offset) +} + +/* Get Torrents with where parameters but no limit and order by default (get all the torrents corresponding in the db) + */ +func GetTorrentsDB(parameters WhereParams) ([]model.Torrents, int) { + return GetTorrentsOrderBy(¶meters, "", 0, 0) +} + +/* Function to get all torrents + */ + +func GetAllTorrentsOrderBy(orderBy string, limit int, offset int) ([]model.Torrents, int) { + + return GetTorrentsOrderBy(nil, orderBy, limit, offset) +} + +func GetAllTorrents(limit int, offset int) ([]model.Torrents, int) { + return GetTorrentsOrderBy(nil, "", limit, offset) +} + +func GetAllTorrentsDB() ([]model.Torrents, int) { + return GetTorrentsOrderBy(nil, "", 0, 0) +} + +/* Function to get all categories with/without torrents (bool) + */ +func GetAllCategories(populatedWithTorrents bool) []model.Categories { + var categories []model.Categories + if populatedWithTorrents { + db.ORM.Preload("Torrents").Preload("Sub_Categories").Find(&categories) + } else { + db.ORM.Preload("Sub_Categories").Find(&categories) + } + return categories +} + +func CreateWhereParams(conditions string, params ...string) WhereParams { + whereParams := WhereParams{} + whereParams.Conditions = conditions + for i, _ := range params { + whereParams.Params = append(whereParams.Params, params[i]) + } + + return whereParams +} \ No newline at end of file diff --git a/templateVariables.go b/templateVariables.go index 937333a0..637009be 100644 --- a/templateVariables.go +++ b/templateVariables.go @@ -2,6 +2,7 @@ package main import ( "github.com/gorilla/mux" + "github.com/ewhal/nyaa/model" "net/url" ) @@ -19,7 +20,7 @@ type FaqTemplateVariables struct { } type ViewTemplateVariables struct { - Torrent TorrentsJson + Torrent model.TorrentsJson Search SearchForm Navigation Navigation URL *url.URL // For parsing Url in templates @@ -27,8 +28,8 @@ type ViewTemplateVariables struct { } type HomeTemplateVariables struct { - ListTorrents []TorrentsJson - ListCategories []Categories + ListTorrents []model.TorrentsJson + ListCategories []model.Categories Search SearchForm Navigation Navigation URL *url.URL // For parsing Url in templates diff --git a/templates/home.html b/templates/home.html index ae84b500..a64d064c 100644 --- a/templates/home.html +++ b/templates/home.html @@ -31,7 +31,7 @@ {{.Name}} - {{.Date}} + {{.Date}} {{.Filesize}} diff --git a/templates/view.html b/templates/view.html index b113c36b..1f05062e 100644 --- a/templates/view.html +++ b/templates/view.html @@ -19,7 +19,7 @@ Date - {{.Date}} + {{.Date}} FileSize diff --git a/util/log/error.go b/util/log/error.go new file mode 100644 index 00000000..f79cad25 --- /dev/null +++ b/util/log/error.go @@ -0,0 +1,29 @@ +package log + +import ( + "fmt" + "github.com/Sirupsen/logrus" + "runtime" + ) + +// CheckError check error and return true if error is nil and return false if error is not nil. +func CheckError(err error) bool { + return CheckErrorWithMessage(err, "") +} + +// CheckErrorWithMessage check error with message and log messages with stack. And then return true if error is nil and return false if error is not nil. +func CheckErrorWithMessage(err error, msg string, args ...interface{}) bool { + if err != nil { + var stack [4096]byte + runtime.Stack(stack[:], false) + // logrus.Errorf(msg, args) + if len(args) == 0 { + logrus.Error(msg + fmt.Sprintf("%q\n%s\n", err, stack[:])) + } else { + logrus.Error(fmt.Sprintf(msg, args...) + fmt.Sprintf("%q\n%s\n", err, stack[:])) + } + // logrus.Printf(msg+"\n%q\n%s\n",args, err, stack[:]) + return false + } + return true +} \ No newline at end of file diff --git a/util/log/logger.go b/util/log/logger.go new file mode 100644 index 00000000..02bdf522 --- /dev/null +++ b/util/log/logger.go @@ -0,0 +1,130 @@ +package log + +import ( + "net/http" + "os" + + "github.com/Sirupsen/logrus" + "github.com/ewhal/nyaa/config" + lumberjack "gopkg.in/natefinch/lumberjack.v2" +) + +func LumberJackLogger(filePath string, maxSize int, maxBackups int, maxAge int) *lumberjack.Logger { + return &lumberjack.Logger{ + Filename: filePath, + MaxSize: maxSize, // megabytes + MaxBackups: maxBackups, + MaxAge: maxAge, //days + } +} + +func InitLogToStdoutDebug() { + logrus.SetFormatter(&logrus.TextFormatter{ForceColors: true}) + logrus.SetOutput(os.Stdout) + logrus.SetLevel(logrus.DebugLevel) +} + +func InitLogToStdout() { + logrus.SetFormatter(&logrus.TextFormatter{}) + logrus.SetOutput(os.Stdout) + logrus.SetLevel(logrus.WarnLevel) +} + +func InitLogToFile() { + logrus.SetFormatter(&logrus.JSONFormatter{}) + + out := LumberJackLogger(config.ErrorLogFilePath+config.ErrorLogFileExtension, config.ErrorLogMaxSize, config.ErrorLogMaxBackups, config.ErrorLogMaxAge) + + + logrus.SetOutput(out) + logrus.SetLevel(logrus.WarnLevel) +} + + + +// Init logrus +func Init(environment string) { + switch environment { + case "DEVELOPMENT": + InitLogToStdoutDebug() + case "TEST": + InitLogToFile() + case "PRODUCTION": + InitLogToFile() + } + logrus.Debugf("Environment : %s", environment) +} + +// Debug logs a message with debug log level. +func Debug(msg string) { + logrus.Debug(msg) +} + +// Debugf logs a formatted message with debug log level. +func Debugf(msg string, args ...interface{}) { + logrus.Debugf(msg, args...) +} + +// Info logs a message with info log level. +func Info(msg string) { + logrus.Info(msg) +} + +// Infof logs a formatted message with info log level. +func Infof(msg string, args ...interface{}) { + logrus.Infof(msg, args...) +} + +// Warn logs a message with warn log level. +func Warn(msg string) { + logrus.Warn(msg) +} + +// Warnf logs a formatted message with warn log level. +func Warnf(msg string, args ...interface{}) { + logrus.Warnf(msg, args...) +} + +// Error logs a message with error log level. +func Error(msg string) { + logrus.Error(msg) +} + +// Errorf logs a formatted message with error log level. +func Errorf(msg string, args ...interface{}) { + logrus.Errorf(msg, args...) +} + +// Fatal logs a message with fatal log level. +func Fatal(msg string) { + logrus.Fatal(msg) +} + +// Fatalf logs a formatted message with fatal log level. +func Fatalf(msg string, args ...interface{}) { + logrus.Fatalf(msg, args...) +} + +// Panic logs a message with panic log level. +func Panic(msg string) { + logrus.Panic(msg) +} + +// Panicf logs a formatted message with panic log level. +func Panicf(msg string, args ...interface{}) { + logrus.Panicf(msg, args...) +} + +// log response body data for debugging +func DebugResponse(response *http.Response) string { + bodyBuffer := make([]byte, 5000) + var str string + count, err := response.Body.Read(bodyBuffer) + for ; count > 0; count, err = response.Body.Read(bodyBuffer) { + if err != nil { + } + str += string(bodyBuffer[:count]) + } + Debugf("response data : %v", str) + return str +} diff --git a/util/safe.go b/util/safe.go new file mode 100644 index 00000000..c7277c35 --- /dev/null +++ b/util/safe.go @@ -0,0 +1,7 @@ +package util + +import "html/template" + +func Safe(s string) template.URL { + return template.URL(s) +} \ No newline at end of file diff --git a/util/unzlib.go b/util/unzlib.go new file mode 100644 index 00000000..eb2ae088 --- /dev/null +++ b/util/unzlib.go @@ -0,0 +1,22 @@ +package util + +import ( + "github.com/ewhal/nyaa/util/log" + + "bytes" + "compress/zlib" + "io/ioutil" +) + +func UnZlib(description []byte) string { + if len(description) > 0 { + b := bytes.NewReader(description) + z, err := zlib.NewReader(b) + log.CheckError(err) + defer z.Close() + p, err := ioutil.ReadAll(z) + log.CheckError(err) + return string(p) + } + return "" +} \ No newline at end of file