From 9586f1e731f720f51abc9d114e18f386fcb4650f Mon Sep 17 00:00:00 2001 From: akuma06 Date: Fri, 5 May 2017 14:20:51 +0200 Subject: [PATCH 01/11] Rearranged Files Configurations are separated in files in the folder config Connection to database by a package to import when needed Models will be in a model package for better maintenance Services access to the models Utils are tools or functions that can be used anywhere main.go cleaned a bit and other files modifications are there for the above modifications --- config/database.go | 5 + config/env.go | 8 ++ config/navigation.go | 6 + config/sorting.go | 7 ++ config/trackers.go | 5 + db/gorm.go | 57 +++++++++ main.go | 115 +++++------------- model/torrent.go | 122 +++++++++++++++++++ models.go | 238 ------------------------------------- service/torrent/torrent.go | 127 ++++++++++++++++++++ templateVariables.go | 7 +- util/log/error.go | 29 +++++ util/safe.go | 7 ++ util/unzlib.go | 22 ++++ 14 files changed, 426 insertions(+), 329 deletions(-) create mode 100644 config/database.go create mode 100644 config/env.go create mode 100644 config/navigation.go create mode 100644 config/sorting.go create mode 100644 config/trackers.go create mode 100644 db/gorm.go create mode 100644 model/torrent.go delete mode 100644 models.go create mode 100644 service/torrent/torrent.go create mode 100644 util/log/error.go create mode 100644 util/safe.go create mode 100644 util/unzlib.go diff --git a/config/database.go b/config/database.go new file mode 100644 index 00000000..349efa34 --- /dev/null +++ b/config/database.go @@ -0,0 +1,5 @@ +package config + +const ( + DbName = "./nyaa.db" +) \ No newline at end of file diff --git a/config/env.go b/config/env.go new file mode 100644 index 00000000..3999a39d --- /dev/null +++ b/config/env.go @@ -0,0 +1,8 @@ +package config + +// Constants for environment. +const ( + // DEVELOPMENT | TEST | PRODUCTION + Environment = "DEVELOPMENT" + // Environment = "PRODUCTION" +) diff --git a/config/navigation.go b/config/navigation.go new file mode 100644 index 00000000..ade0972d --- /dev/null +++ b/config/navigation.go @@ -0,0 +1,6 @@ +package config + +// Constants for pagination. +const ( + TorrentsPerPage = 50 +) diff --git a/config/sorting.go b/config/sorting.go new file mode 100644 index 00000000..41df588a --- /dev/null +++ b/config/sorting.go @@ -0,0 +1,7 @@ +package config + +// Constants for ordering models. +const ( + TorrentOrder = "torrent_id" + TorrentSort = "DESC" +) diff --git a/config/trackers.go b/config/trackers.go new file mode 100644 index 00000000..ed0eed33 --- /dev/null +++ b/config/trackers.go @@ -0,0 +1,5 @@ +package config + +const ( + Trackers = "&tr=udp://zer0day.to:1337/announce&tr=udp://tracker.leechers-paradise.org:6969&tr=udp://explodie.org:6969&tr=udp://tracker.opentrackr.org:1337&tr=udp://tracker.coppersurfer.tk:6969&tr=http://tracker.baka-sub.cf/announce" +) \ No newline at end of file diff --git a/db/gorm.go b/db/gorm.go new file mode 100644 index 00000000..3b159113 --- /dev/null +++ b/db/gorm.go @@ -0,0 +1,57 @@ +package db + +import ( + "github.com/ewhal/nyaa/config" + "github.com/ewhal/nyaa/util/log" + "github.com/ewhal/nyaa/model" + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/sqlite" + // _ "github.com/go-sql-driver/mysql" +) + +var ORM, Errs = GormInit() + +// GormInit init gorm ORM. +func GormInit() (*gorm.DB, error) { + db, err := gorm.Open("sqlite3", config.DbName) + // db, err := gorm.Open("mysql", config.MysqlDSL()) + //db, err := gorm.Open("sqlite3", "/tmp/gorm.db") + + // Get database connection handle [*sql.DB](http://golang.org/pkg/database/sql/#DB) + db.DB() + + // Then you could invoke `*sql.DB`'s functions with it + db.DB().Ping() + db.DB().SetMaxIdleConns(10) + db.DB().SetMaxOpenConns(100) + + // Disable table name's pluralization + // db.SingularTable(true) + if config.Environment == "DEVELOPMENT" { + db.LogMode(true) + // db.DropTable(&model.User{}, "UserFollower") + db.AutoMigrate(&model.Torrents{}, &model.Categories{}, &model.Sub_Categories{}, &model.Statuses{}) + // db.AutoMigrate(&model.User{}, &model.Role{}, &model.Connection{}, &model.Language{}, &model.Article{}, &model.Location{}, &model.Comment{}, &model.File{}) + // db.Model(&model.User{}).AddIndex("idx_user_token", "token") + + } + log.CheckError(err) + + // relation := gorm.Relationship{} + // relation.Kind = "many2many" + // relation.ForeignFieldNames = []string{"id"} //(M1 pkey) + // relation.ForeignDBNames = []string{"user_id"} //(M1 fkey in m1m2join) + // relation.AssociationForeignFieldNames = []string{"id"} //(M2 pkey) + // // relation.AssociationForeignStructFieldNames = []string{"id", "ID"} //(m2 pkey name in m2 struct?) + // relation.AssociationForeignDBNames = []string{"follower_id"} //(m2 fkey in m1m2join) + // m1Type := reflect.TypeOf(model.User{}) + // m2Type := reflect.TypeOf(model.User{}) + // handler := gorm.JoinTableHandler{} + // // ORDER BELOW MATTERS + // // Install handler + // db.SetJoinTableHandler(&model.User{}, "Likings", &handler) + // // Configure handler to use the relation that we've defined + // handler.Setup(&relation, "users_followers", m1Type, m2Type) + + return db, err +} diff --git a/main.go b/main.go index 9d69cc98..254cab4e 100644 --- a/main.go +++ b/main.go @@ -1,57 +1,23 @@ package main import ( - "bytes" - "compress/zlib" "encoding/json" "github.com/gorilla/feeds" "github.com/gorilla/mux" - "github.com/jinzhu/gorm" - _ "github.com/jinzhu/gorm/dialects/sqlite" + + "github.com/ewhal/nyaa/model" + "github.com/ewhal/nyaa/service/torrent" + "github.com/ewhal/nyaa/util/log" + "html" "html/template" - "io/ioutil" - "log" "net/http" "strconv" "strings" "time" ) -var db *gorm.DB var router *mux.Router -var debugLogger *log.Logger -var trackers = "&tr=udp://zer0day.to:1337/announce&tr=udp://tracker.leechers-paradise.org:6969&tr=udp://explodie.org:6969&tr=udp://tracker.opentrackr.org:1337&tr=udp://tracker.coppersurfer.tk:6969&tr=http://tracker.baka-sub.cf/announce" - -func getDBHandle() *gorm.DB { - dbInit, err := gorm.Open("sqlite3", "./nyaa.db") - - // Migrate the schema of Torrents - dbInit.AutoMigrate(&Torrents{}, &Categories{}, &Sub_Categories{}, &Statuses{}) - - checkErr(err) - return dbInit -} - -func checkErr(err error) { - if err != nil { - debugLogger.Println(" " + err.Error()) - } -} - -func unZlib(description []byte) string { - if len(description) > 0 { - b := bytes.NewReader(description) - //log.Println(b) - z, err := zlib.NewReader(b) - checkErr(err) - defer z.Close() - p, err := ioutil.ReadAll(z) - checkErr(err) - return string(p) - } - return "" -} func apiHandler(w http.ResponseWriter, r *http.Request) { @@ -59,13 +25,13 @@ func apiHandler(w http.ResponseWriter, r *http.Request) { page := vars["page"] pagenum, _ := strconv.Atoi(html.EscapeString(page)) - b := CategoryJson{Torrents: []TorrentsJson{}} + b := model.CategoryJson{Torrents: []model.TorrentsJson{}} maxPerPage := 50 nbTorrents := 0 - torrents, nbTorrents := getAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) + torrents, nbTorrents := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) for i, _ := range torrents { - res := torrents[i].toJson() + res := torrents[i].ToJson() b.Torrents = append(b.Torrents, res) } @@ -84,10 +50,10 @@ func apiViewHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id := vars["id"] - b := CategoryJson{Torrents: []TorrentsJson{}} + b := model.CategoryJson{Torrents: []model.TorrentsJson{}} - torrent, err := getTorrentById(id) - res := torrent.toJson() + torrent, err := torrentService.GetTorrentById(id) + res := torrent.ToJson() b.Torrents = append(b.Torrents, res) b.QueryRecordCount = 1 @@ -103,7 +69,6 @@ func apiViewHandler(w http.ResponseWriter, r *http.Request) { func searchHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.New("home").Funcs(funcMap).ParseFiles("templates/index.html", "templates/home.html")) - templates.ParseGlob("templates/_*.html") // common vars := mux.Vars(r) page := vars["page"] @@ -126,6 +91,7 @@ func searchHandler(w http.ResponseWriter, r *http.Request) { // need this to prevent out of index panics var searchCatId, searchSubCatId string if len(catsSplit) == 2 { + searchCatId = html.EscapeString(catsSplit[0]) searchSubCatId = html.EscapeString(catsSplit[1]) } @@ -139,53 +105,29 @@ func searchHandler(w http.ResponseWriter, r *http.Request) { nbTorrents := 0 - b := []TorrentsJson{} + b := []model.TorrentsJson{} - parameters := WhereParams{} - conditions := []string{} - if searchCatId != "" { - conditions = append(conditions, "category_id = ?") - parameters.params = append(parameters.params, searchCatId) - } - if searchSubCatId != "" { - conditions = append(conditions, "sub_category_id = ?") - parameters.params = append(parameters.params, searchSubCatId) - } - if stat != "" { - conditions = append(conditions, "status_id = ?") - parameters.params = append(parameters.params, stat) - } - searchQuerySplit := strings.Split(searchQuery, " ") - for i, _ := range searchQuerySplit { - conditions = append(conditions, "torrent_name LIKE ?") - parameters.params = append(parameters.params, "%"+searchQuerySplit[i]+"%") - } - - parameters.conditions = strings.Join(conditions[:], " AND ") - log.Printf("SQL query is :: %s\n", parameters.conditions) - torrents, nbTorrents := getTorrentsOrderBy(¶meters, order_by, maxPerPage, maxPerPage*(pagenum-1)) + parameters := torrentService.CreateWhereParams("torrent_name LIKE ? AND status_id LIKE ? AND category_id LIKE ? AND sub_category_id LIKE ?", + "%"+searchQuery+"%", stat+"%", searchCatId+"%", searchSubCatId+"%") + torrents, nbTorrents := torrentService.GetTorrentsOrderBy(¶meters, order_by, maxPerPage, maxPerPage*(pagenum-1)) for i, _ := range torrents { - res := torrents[i].toJson() + res := torrents[i].ToJson() b = append(b, res) } navigationTorrents := Navigation{nbTorrents, maxPerPage, pagenum, "search_page"} searchForm := SearchForm{searchQuery, stat, cat, sort, order} - htv := HomeTemplateVariables{b, getAllCategories(false), searchForm, navigationTorrents, r.URL, mux.CurrentRoute(r)} + htv := HomeTemplateVariables{b, torrentService.GetAllCategories(false), searchForm, navigationTorrents, r.URL, mux.CurrentRoute(r)} err := templates.ExecuteTemplate(w, "index.html", htv) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } } -func safe(s string) template.URL { - return template.URL(s) -} func faqHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.New("FAQ").Funcs(funcMap).ParseFiles("templates/index.html", "templates/FAQ.html")) - templates.ParseGlob("templates/_*.html") // common err := templates.ExecuteTemplate(w, "index.html", FaqTemplateVariables{Navigation{}, NewSearchForm(), r.URL, mux.CurrentRoute(r)}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -199,7 +141,7 @@ func rssHandler(w http.ResponseWriter, r *http.Request) { // db params url //maxPerPage := 50 // default Value maxPerPage - torrents := getFeeds() + torrents := torrentService.GetFeeds() created := time.Now().String() if len(torrents) > 0 { created = torrents[0].Timestamp @@ -240,12 +182,11 @@ func rssHandler(w http.ResponseWriter, r *http.Request) { } func viewHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.ParseFiles("templates/index.html", "templates/view.html")) - templates.ParseGlob("templates/_*.html") // common vars := mux.Vars(r) id := vars["id"] - torrent, err := getTorrentById(id) - b := torrent.toJson() + torrent, err := torrentService.GetTorrentById(id) + b := torrent.ToJson() htv := ViewTemplateVariables{b, NewSearchForm(), Navigation{}, r.URL, mux.CurrentRoute(r)} @@ -257,7 +198,6 @@ func viewHandler(w http.ResponseWriter, r *http.Request) { func rootHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.New("home").Funcs(funcMap).ParseFiles("templates/index.html", "templates/home.html")) - templates.ParseGlob("templates/_*.html") // common vars := mux.Vars(r) page := vars["page"] @@ -273,16 +213,16 @@ func rootHandler(w http.ResponseWriter, r *http.Request) { pagenum = 1 } - b := []TorrentsJson{} - torrents, nbTorrents := getAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) + b := []model.TorrentsJson{} + torrents, nbTorrents := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) for i, _ := range torrents { - res := torrents[i].toJson() + res := torrents[i].ToJson() b = append(b, res) } navigationTorrents := Navigation{nbTorrents, maxPerPage, pagenum, "search_page"} - htv := HomeTemplateVariables{b, getAllCategories(false), NewSearchForm(), navigationTorrents, r.URL, mux.CurrentRoute(r)} + htv := HomeTemplateVariables{b, torrentService.GetAllCategories(false), NewSearchForm(), navigationTorrents, r.URL, mux.CurrentRoute(r)} err := templates.ExecuteTemplate(w, "index.html", htv) if err != nil { @@ -293,7 +233,6 @@ func rootHandler(w http.ResponseWriter, r *http.Request) { func main() { - db = getDBHandle() router = mux.NewRouter() cssHandler := http.FileServer(http.Dir("./css/")) @@ -324,5 +263,5 @@ func main() { } err := srv.ListenAndServe() - checkErr(err) -} + log.CheckError(err) +} \ No newline at end of file diff --git a/model/torrent.go b/model/torrent.go new file mode 100644 index 00000000..58f0aeb3 --- /dev/null +++ b/model/torrent.go @@ -0,0 +1,122 @@ +package model + +import ( + "github.com/ewhal/nyaa/util" + "github.com/ewhal/nyaa/config" + + "html" + "html/template" + "strconv" + "strings" + "time" +) + +type Feed struct { + Id int + Name string + Hash string + Magnet string + Timestamp string +} + +type Categories struct { + Id int `gorm:"column:category_id"` + Name string `gorm:"column:category_name"` + Torrents []Torrents `gorm:"ForeignKey:category_id;AssociationForeignKey:category_id"` + Sub_Categories []Sub_Categories `gorm:"ForeignKey:parent_id;AssociationForeignKey:category_id"` +} + +type Sub_Categories struct { + Id int `gorm:"column:sub_category_id"` + Name string `gorm:"column:Sub_category_name"` + Parent_id int `gorm:"column:parent_id"` + Torrents []Torrents `gorm:"ForeignKey:sub_category_id;AssociationForeignKey:sub_category_id"` +} + +type Statuses struct { + Status_id int + Status_name string + Torrents []Torrents `gorm:"ForeignKey:status_id;AssociationForeignKey:status_id"` +} + +type Torrents struct { + Id int `gorm:"column:torrent_id"` + Name string `gorm:"column:torrent_name"` + Category_id int `gorm:"column:category_id"` + Sub_category_id int `gorm:"column:sub_category_id"` + Status int `gorm:"column:status_id"` + Hash string `gorm:"column:torrent_hash"` + Date int64 `gorm:"column:date"` + Downloads int `gorm:"column:downloads"` + Filesize string `gorm:"column:filesize"` + Description []byte `gorm:"column:description"` + Statuses Statuses `gorm:"ForeignKey:status_id;AssociationForeignKey:status_id"` + Categories Categories `gorm:"ForeignKey:category_id;AssociationForeignKey:category_id"` + Sub_Categories Sub_Categories `gorm:"ForeignKey:sub_category_id;AssociationForeignKey:sub_category_id"` +} + +/* We need JSON Object instead because of Magnet URL that is not in the database but generated dynamically +-------------------------------------------------------------------------------------------------------------- +JSON Models Oject +-------------------------------------------------------------------------------------------------------------- +*/ + +type CategoryJson struct { + Id string `json: "id"` + Name string `json: "category"` + Torrents []TorrentsJson `json: "torrents"` + QueryRecordCount int `json: "queryRecordCount"` + TotalRecordCount int `json: "totalRecordCount"` +} + +type SubCategoryJson struct { + Id string `json: "id"` + Name string `json: "category"` +} + +type TorrentsJson struct { + Id string `json: "id"` // Is there a need to put the ID? + Name string `json: "name"` + Status int `json: "status"` + Hash string `json: "hash"` + Date string `json: "date"` + Filesize string `json: "filesize"` + Description template.HTML `json: "description"` + Sub_Category SubCategoryJson `json: "sub_category"` + Category CategoryJson `json: "category"` + Magnet template.URL `json: "magnet"` +} + + +/* Model Conversion to Json */ + +func (t *Torrents) ToJson() TorrentsJson { + magnet := "magnet:?xt=urn:btih:" + strings.TrimSpace(t.Hash) + "&dn=" + t.Name + config.Trackers + res := TorrentsJson{ + Id: strconv.Itoa(t.Id), + Name: html.UnescapeString(t.Name), + Status: t.Status, + Hash: t.Hash, + Date: time.Unix(t.Date, 0).Format(time.RFC3339), + Filesize: t.Filesize, + Description: template.HTML(util.UnZlib(t.Description)), + Sub_Category: t.Sub_Categories.ToJson(), + Category: t.Categories.ToJson(), + Magnet: util.Safe(magnet)} + + return res +} + +func (c *Sub_Categories) ToJson() SubCategoryJson { + return SubCategoryJson{ + Id: strconv.Itoa(c.Id), + Name: html.UnescapeString(c.Name)} +} + +func (c *Categories) ToJson() CategoryJson { + return CategoryJson{ + Id: strconv.Itoa(c.Id), + Name: html.UnescapeString(c.Name)} +} + +/* Complete the functions when necessary... */ diff --git a/models.go b/models.go deleted file mode 100644 index 8580932c..00000000 --- a/models.go +++ /dev/null @@ -1,238 +0,0 @@ -package main - -import ( - "errors" - "github.com/jinzhu/gorm" - "html" - "html/template" - "strconv" - "strings" - "time" -) - -type Feed struct { - Id int - Name string - Hash string - Magnet string - Timestamp string -} - -type Categories struct { - Id int `gorm:"column:category_id"` - Name string `gorm:"column:category_name"` - Torrents []Torrents `gorm:"ForeignKey:category_id;AssociationForeignKey:category_id"` - Sub_Categories []Sub_Categories `gorm:"ForeignKey:parent_id;AssociationForeignKey:category_id"` -} - -type Sub_Categories struct { - Id int `gorm:"column:sub_category_id"` - Name string `gorm:"column:Sub_category_name"` - Parent_id int `gorm:"column:parent_id"` - Torrents []Torrents `gorm:"ForeignKey:sub_category_id;AssociationForeignKey:sub_category_id"` -} - -type Statuses struct { - Status_id int - Status_name string - Torrents []Torrents `gorm:"ForeignKey:status_id;AssociationForeignKey:status_id"` -} - -type Torrents struct { - Id int `gorm:"column:torrent_id"` - Name string `gorm:"column:torrent_name"` - Category_id int `gorm:"column:category_id"` - Sub_category_id int `gorm:"column:sub_category_id"` - Status int `gorm:"column:status_id"` - Hash string `gorm:"column:torrent_hash"` - Date int64 `gorm:"column:date"` - Downloads int `gorm:"column:downloads"` - Filesize string `gorm:"column:filesize"` - Description []byte `gorm:"column:description"` - Statuses Statuses `gorm:"ForeignKey:status_id;AssociationForeignKey:status_id"` - Categories Categories `gorm:"ForeignKey:category_id;AssociationForeignKey:category_id"` - Sub_Categories Sub_Categories `gorm:"ForeignKey:sub_category_id;AssociationForeignKey:sub_category_id"` -} - -/* We need JSON Object instead because of Magnet URL that is not in the database but generated dynamically --------------------------------------------------------------------------------------------------------------- -JSON Models Oject --------------------------------------------------------------------------------------------------------------- -*/ - -type CategoryJson struct { - Id string `json: "id"` - Name string `json: "category"` - Torrents []TorrentsJson `json: "torrents"` - QueryRecordCount int `json: "queryRecordCount"` - TotalRecordCount int `json: "totalRecordCount"` -} - -type SubCategoryJson struct { - Id string `json: "id"` - Name string `json: "category"` -} - -type TorrentsJson struct { - Id string `json: "id"` // Is there a need to put the ID? - Name string `json: "name"` - Status int `json: "status"` - Hash string `json: "hash"` - Date string `json: "date"` - Filesize string `json: "filesize"` - Description template.HTML `json: "description"` - Sub_Category SubCategoryJson `json: "sub_category"` - Category CategoryJson `json: "category"` - Magnet template.URL `json: "magnet"` -} - -type WhereParams struct { - conditions string // Ex : name LIKE ? AND category_id LIKE ? - params []interface{} -} - -/* Function to interact with Models - * - * Get the torrents with where clause - * - */ - -// don't need raw SQL once we get MySQL -func getFeeds() []Feed { - var result []Feed - rows, err := db.DB(). - Query( - "SELECT `torrent_id` AS `id`, `torrent_name` AS `name`, `torrent_hash` AS `hash`, `timestamp` FROM `torrents` " + - "ORDER BY `timestamp` desc LIMIT 50") - if err == nil { - for rows.Next() { - item := Feed{} - rows.Scan(&item.Id, &item.Name, &item.Hash, &item.Timestamp) - magnet := "magnet:?xt=urn:btih:" + strings.TrimSpace(item.Hash) + "&dn=" + item.Name + trackers - item.Magnet = magnet - // memory hog - result = append(result, item) - } - rows.Close() - } - return result -} - -func getTorrentById(id string) (Torrents, error) { - var torrent Torrents - - if db.Where("torrent_id = ?", id).Find(&torrent).RecordNotFound() { - return torrent, errors.New("Article is not found.") - } - - return torrent, nil -} - -func getTorrentsOrderBy(parameters *WhereParams, orderBy string, limit int, offset int) ([]Torrents, int) { - var torrents []Torrents - var dbQuery *gorm.DB - var count int - conditions := "torrent_hash is not null" //filter out broken entries - var params []interface{} - if parameters != nil { // if there is where parameters - conditions += " AND " + parameters.conditions - params = parameters.params - } - db.Model(&torrents).Where(conditions, params...).Count(&count) - dbQuery = db.Model(&torrents).Where(conditions, params...) - - if orderBy == "" { - orderBy = "torrent_id DESC" - } // Default OrderBy - if limit != 0 || offset != 0 { // if limits provided - dbQuery = dbQuery.Limit(limit).Offset(offset) - } - dbQuery.Order(orderBy).Preload("Categories").Preload("Sub_Categories").Find(&torrents) - return torrents, count - -} - -/* Functions to simplify the get parameters of the main function - * - * Get Torrents with where parameters and limits, order by default - */ -func getTorrents(parameters WhereParams, limit int, offset int) ([]Torrents, int) { - return getTorrentsOrderBy(¶meters, "", limit, offset) -} - -/* Get Torrents with where parameters but no limit and order by default (get all the torrents corresponding in the db) - */ -func getTorrentsDB(parameters WhereParams) ([]Torrents, int) { - return getTorrentsOrderBy(¶meters, "", 0, 0) -} - -/* Function to get all torrents - */ - -func getAllTorrentsOrderBy(orderBy string, limit int, offset int) ([]Torrents, int) { - - return getTorrentsOrderBy(nil, orderBy, limit, offset) -} - -func getAllTorrents(limit int, offset int) ([]Torrents, int) { - return getTorrentsOrderBy(nil, "", limit, offset) -} - -func getAllTorrentsDB() ([]Torrents, int) { - return getTorrentsOrderBy(nil, "", 0, 0) -} - -/* Function to get all categories with/without torrents (bool) - */ -func getAllCategories(populatedWithTorrents bool) []Categories { - var categories []Categories - if populatedWithTorrents { - db.Preload("Torrents").Preload("Sub_Categories").Find(&categories) - } else { - db.Preload("Sub_Categories").Find(&categories) - } - return categories -} - -func createWhereParams(conditions string, params ...string) WhereParams { - whereParams := WhereParams{} - whereParams.conditions = conditions - for i, _ := range params { - whereParams.params = append(whereParams.params, params[i]) - } - - return whereParams -} - -/* Model Conversion to Json */ - -func (t *Torrents) toJson() TorrentsJson { - magnet := "magnet:?xt=urn:btih:" + strings.TrimSpace(t.Hash) + "&dn=" + t.Name + trackers - res := TorrentsJson{ - Id: strconv.Itoa(t.Id), - Name: html.UnescapeString(t.Name), - Status: t.Status, - Hash: t.Hash, - Date: time.Unix(t.Date, 0).Format(time.RFC3339), - Filesize: t.Filesize, - Description: template.HTML(unZlib(t.Description)), - Sub_Category: t.Sub_Categories.toJson(), - Category: t.Categories.toJson(), - Magnet: safe(magnet)} - - return res -} - -func (c *Sub_Categories) toJson() SubCategoryJson { - return SubCategoryJson{ - Id: strconv.Itoa(c.Id), - Name: html.UnescapeString(c.Name)} -} - -func (c *Categories) toJson() CategoryJson { - return CategoryJson{ - Id: strconv.Itoa(c.Id), - Name: html.UnescapeString(c.Name)} -} - -/* Complete the functions when necessary... */ diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go new file mode 100644 index 00000000..ea8841ad --- /dev/null +++ b/service/torrent/torrent.go @@ -0,0 +1,127 @@ +package torrentService +import ( + "github.com/ewhal/nyaa/db" + "github.com/ewhal/nyaa/model" + "github.com/ewhal/nyaa/config" + "github.com/jinzhu/gorm" + "errors" + "strings" +) + +type WhereParams struct { + conditions string // Ex : name LIKE ? AND category_id LIKE ? + params []interface{} +} + +/* Function to interact with Models + * + * Get the torrents with where clause + * + */ + +// don't need raw SQL once we get MySQL +func GetFeeds() []model.Feed { + var result []model.Feed + rows, err := db.ORM.DB(). + Query( + "SELECT `torrent_id` AS `id`, `torrent_name` AS `name`, `torrent_hash` AS `hash`, `timestamp` FROM `torrents` " + + "ORDER BY `timestamp` desc LIMIT 50") + if err == nil { + for rows.Next() { + item := model.Feed{} + rows.Scan(&item.Id, &item.Name, &item.Hash, &item.Timestamp) + magnet := "magnet:?xt=urn:btih:" + strings.TrimSpace(item.Hash) + "&dn=" + item.Name + config.Trackers + item.Magnet = magnet + // memory hog + result = append(result, item) + } + rows.Close() + } + return result +} + +func GetTorrentById(id string) (model.Torrents, error) { + var torrent model.Torrents + + if db.ORM.Where("torrent_id = ?", id).Find(&torrent).RecordNotFound() { + return torrent, errors.New("Article is not found.") + } + + return torrent, nil +} + +func GetTorrentsOrderBy(parameters *WhereParams, orderBy string, limit int, offset int) ([]model.Torrents, int) { + var torrents []model.Torrents + var dbQuery *gorm.DB + var count int + conditions := "torrent_hash is not null" //filter out broken entries + var params []interface{} + if parameters != nil { // if there is where parameters + conditions += " AND " + parameters.conditions + params = parameters.params + } + db.ORM.Model(&torrents).Where(conditions, params...).Count(&count) + dbQuery = db.ORM.Model(&torrents).Where(conditions, params...) + + if orderBy == "" { + orderBy = "torrent_id DESC" + } // Default OrderBy + if limit != 0 || offset != 0 { // if limits provided + dbQuery = dbQuery.Limit(limit).Offset(offset) + } + dbQuery.Order(orderBy).Preload("Categories").Preload("Sub_Categories").Find(&torrents) + return torrents, count + +} + +/* Functions to simplify the get parameters of the main function + * + * Get Torrents with where parameters and limits, order by default + */ +func GetTorrents(parameters WhereParams, limit int, offset int) ([]model.Torrents, int) { + return GetTorrentsOrderBy(¶meters, "", limit, offset) +} + +/* Get Torrents with where parameters but no limit and order by default (get all the torrents corresponding in the db) + */ +func GetTorrentsDB(parameters WhereParams) ([]model.Torrents, int) { + return GetTorrentsOrderBy(¶meters, "", 0, 0) +} + +/* Function to get all torrents + */ + +func GetAllTorrentsOrderBy(orderBy string, limit int, offset int) ([]model.Torrents, int) { + + return GetTorrentsOrderBy(nil, orderBy, limit, offset) +} + +func GetAllTorrents(limit int, offset int) ([]model.Torrents, int) { + return GetTorrentsOrderBy(nil, "", limit, offset) +} + +func GetAllTorrentsDB() ([]model.Torrents, int) { + return GetTorrentsOrderBy(nil, "", 0, 0) +} + +/* Function to get all categories with/without torrents (bool) + */ +func GetAllCategories(populatedWithTorrents bool) []model.Categories { + var categories []model.Categories + if populatedWithTorrents { + db.ORM.Preload("Torrents").Preload("Sub_Categories").Find(&categories) + } else { + db.ORM.Preload("Sub_Categories").Find(&categories) + } + return categories +} + +func CreateWhereParams(conditions string, params ...string) WhereParams { + whereParams := WhereParams{} + whereParams.conditions = conditions + for i, _ := range params { + whereParams.params = append(whereParams.params, params[i]) + } + + return whereParams +} \ No newline at end of file diff --git a/templateVariables.go b/templateVariables.go index 937333a0..637009be 100644 --- a/templateVariables.go +++ b/templateVariables.go @@ -2,6 +2,7 @@ package main import ( "github.com/gorilla/mux" + "github.com/ewhal/nyaa/model" "net/url" ) @@ -19,7 +20,7 @@ type FaqTemplateVariables struct { } type ViewTemplateVariables struct { - Torrent TorrentsJson + Torrent model.TorrentsJson Search SearchForm Navigation Navigation URL *url.URL // For parsing Url in templates @@ -27,8 +28,8 @@ type ViewTemplateVariables struct { } type HomeTemplateVariables struct { - ListTorrents []TorrentsJson - ListCategories []Categories + ListTorrents []model.TorrentsJson + ListCategories []model.Categories Search SearchForm Navigation Navigation URL *url.URL // For parsing Url in templates diff --git a/util/log/error.go b/util/log/error.go new file mode 100644 index 00000000..f79cad25 --- /dev/null +++ b/util/log/error.go @@ -0,0 +1,29 @@ +package log + +import ( + "fmt" + "github.com/Sirupsen/logrus" + "runtime" + ) + +// CheckError check error and return true if error is nil and return false if error is not nil. +func CheckError(err error) bool { + return CheckErrorWithMessage(err, "") +} + +// CheckErrorWithMessage check error with message and log messages with stack. And then return true if error is nil and return false if error is not nil. +func CheckErrorWithMessage(err error, msg string, args ...interface{}) bool { + if err != nil { + var stack [4096]byte + runtime.Stack(stack[:], false) + // logrus.Errorf(msg, args) + if len(args) == 0 { + logrus.Error(msg + fmt.Sprintf("%q\n%s\n", err, stack[:])) + } else { + logrus.Error(fmt.Sprintf(msg, args...) + fmt.Sprintf("%q\n%s\n", err, stack[:])) + } + // logrus.Printf(msg+"\n%q\n%s\n",args, err, stack[:]) + return false + } + return true +} \ No newline at end of file diff --git a/util/safe.go b/util/safe.go new file mode 100644 index 00000000..c7277c35 --- /dev/null +++ b/util/safe.go @@ -0,0 +1,7 @@ +package util + +import "html/template" + +func Safe(s string) template.URL { + return template.URL(s) +} \ No newline at end of file diff --git a/util/unzlib.go b/util/unzlib.go new file mode 100644 index 00000000..eb2ae088 --- /dev/null +++ b/util/unzlib.go @@ -0,0 +1,22 @@ +package util + +import ( + "github.com/ewhal/nyaa/util/log" + + "bytes" + "compress/zlib" + "io/ioutil" +) + +func UnZlib(description []byte) string { + if len(description) > 0 { + b := bytes.NewReader(description) + z, err := zlib.NewReader(b) + log.CheckError(err) + defer z.Close() + p, err := ioutil.ReadAll(z) + log.CheckError(err) + return string(p) + } + return "" +} \ No newline at end of file From d1e511c480df7c958940e04bf8b9a1f4fb6c5c3b Mon Sep 17 00:00:00 2001 From: Adelyne Maulideau Date: Fri, 5 May 2017 14:54:36 +0200 Subject: [PATCH 02/11] json.MarshalIndent's error was checked after trying to use its value. --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 33b8f418..0d1fd054 100644 --- a/config.go +++ b/config.go @@ -96,10 +96,10 @@ func (config *Config) Write(output io.Writer) error { func (config *Config) Pretty(output io.Writer) error { data, err := json.MarshalIndent(config, "", "\t") - data = append(data, []byte("\n")...) if err != nil { return err } + data = append(data, []byte("\n")...) _, err = output.Write(data) return err } From 46c0025ee36311f4a4947ecba74a187d68c2823e Mon Sep 17 00:00:00 2001 From: akuma06 Date: Fri, 5 May 2017 15:13:45 +0200 Subject: [PATCH 03/11] Forgot to add go get --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6815c806..c5eb2a98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ install: - go get github.com/gorilla/mux - go get github.com/mattn/go-sqlite3 - go get github.com/jinzhu/gorm +- go get github.com/Sirupsen/logrus +- go get gopkg.in/natefinch/lumberjack.v2 - go build deploy: provider: releases From a70551a986b1ae5f4c840f9e18895d129655d0c9 Mon Sep 17 00:00:00 2001 From: akuma06 Date: Fri, 5 May 2017 15:23:49 +0200 Subject: [PATCH 04/11] Re added templates files function for search Removed them when I've resolved the conflicts --- main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 9c0db60e..6c6db191 100644 --- a/main.go +++ b/main.go @@ -82,6 +82,7 @@ func apiViewHandler(w http.ResponseWriter, r *http.Request) { func searchHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.New("home").Funcs(funcMap).ParseFiles("templates/index.html", "templates/home.html")) + templates.ParseGlob("templates/_*.html") // common vars := mux.Vars(r) page := vars["page"] @@ -174,6 +175,7 @@ func searchByQuery(r *http.Request, pagenum int) (SearchParam, []model.Torrents, func faqHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.New("FAQ").Funcs(funcMap).ParseFiles("templates/index.html", "templates/FAQ.html")) + templates.ParseGlob("templates/_*.html") // common err := templates.ExecuteTemplate(w, "index.html", FaqTemplateVariables{Navigation{}, NewSearchForm(), r.URL, mux.CurrentRoute(r)}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -220,6 +222,7 @@ func rssHandler(w http.ResponseWriter, r *http.Request) { func viewHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.ParseFiles("templates/index.html", "templates/view.html")) + templates.ParseGlob("templates/_*.html") // common vars := mux.Vars(r) id := vars["id"] @@ -235,8 +238,9 @@ func viewHandler(w http.ResponseWriter, r *http.Request) { } func rootHandler(w http.ResponseWriter, r *http.Request) { - var templates = template.Must(template.New("home").Funcs(funcMap).ParseFiles("templates/index.html", "templates/home.html")) - vars := mux.Vars(r) +var templates = template.Must(template.New("home").Funcs(funcMap).ParseFiles("templates/index.html", "templates/home.html")) + templates.ParseGlob("templates/_*.html") // common + vars := mux.Vars(r) page := vars["page"] // db params url From 1d8246b5565d59b3b0e64f7505d451ead8f76f94 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 5 May 2017 15:00:07 +0200 Subject: [PATCH 05/11] Style fixes --- css/style.css | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/css/style.css b/css/style.css index 406f043a..5dd91a5c 100644 --- a/css/style.css +++ b/css/style.css @@ -16,7 +16,7 @@ nav#mainmenu { position: fixed; color: white; width: 100%; - z-index: 3; + z-index: 4; } nav#mainmenu a { @@ -72,6 +72,11 @@ a { white-space: nowrap; } +.table > tbody > tr > td { + /* fix bootstrap uglyness */ + vertical-align: middle; +} + div.container div.blockBody:nth-of-type(2) table{table-layout:fixed;} div.container div.blockBody:nth-of-type(2) table tr:first-of-type th:first-of-type, div.container div.blockBody:nth-of-type(2) table tr:first-of-type th:nth-of-type(5){width:10%;} div.container div.blockBody:nth-of-type(2) table tr:first-of-type th:nth-of-type(3){width:15%;} From f9714e8f87e74b75a03ef9acfc47e8d81636f72a Mon Sep 17 00:00:00 2001 From: Eliot Whalan Date: Fri, 5 May 2017 23:36:10 +1000 Subject: [PATCH 06/11] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aeeed88f..28f5b306 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,12 @@ The provided unit file uses options directly; if you prefer a config file, do th * Replace in the unit file the options by `-conf /etc/nyaa.conf` ## TODO -* RSS feeds(work in progress) -* torrent sorting (work in progress) +* uploading * API improvement +* improve view page +* make sukebei db schema compatible with current code * Site theme -* Torrent view and description page(work in progress) +* comments in torrent view page * accounts? * Adding new torrents * scraping From a23f62e08c9cf6aa0398bcecbb07a59dc67e9eb5 Mon Sep 17 00:00:00 2001 From: Eliot Whalan Date: Fri, 5 May 2017 23:37:40 +1000 Subject: [PATCH 07/11] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 28f5b306..51c32254 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ The provided unit file uses options directly; if you prefer a config file, do th ## TODO * uploading * API improvement +* add rss link and generate link based on your current search * improve view page * make sukebei db schema compatible with current code * Site theme From 365c78194bcb4951da8dc35bcc24bcbc8d58b604 Mon Sep 17 00:00:00 2001 From: Eliot Whalan Date: Fri, 5 May 2017 23:38:02 +1000 Subject: [PATCH 08/11] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 51c32254..b01c24fb 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The provided unit file uses options directly; if you prefer a config file, do th * Replace in the unit file the options by `-conf /etc/nyaa.conf` ## TODO -* uploading +* uploading/adding new magnet links * API improvement * add rss link and generate link based on your current search * improve view page @@ -42,7 +42,6 @@ The provided unit file uses options directly; if you prefer a config file, do th * Site theme * comments in torrent view page * accounts? -* Adding new torrents * scraping * Daily DB dumps * p2p sync of dbs? From ccf8e938f8eb66795e070cc9acbda708a922cc06 Mon Sep 17 00:00:00 2001 From: Eliot Whalan Date: Fri, 5 May 2017 23:39:18 +1000 Subject: [PATCH 09/11] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b01c24fb..698f2eb6 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ The provided unit file uses options directly; if you prefer a config file, do th * API improvement * add rss link and generate link based on your current search * improve view page +* make renchon the favicon and official mascot * make sukebei db schema compatible with current code * Site theme * comments in torrent view page From f336dcfc68ed694c249c0fbcfe0e4254d10e51b4 Mon Sep 17 00:00:00 2001 From: Eliot Whalan Date: Fri, 5 May 2017 23:43:26 +1000 Subject: [PATCH 10/11] Update README.md --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 698f2eb6..828247fd 100644 --- a/README.md +++ b/README.md @@ -34,14 +34,13 @@ The provided unit file uses options directly; if you prefer a config file, do th * Replace in the unit file the options by `-conf /etc/nyaa.conf` ## TODO -* uploading/adding new magnet links +* adding new magnet links * API improvement * add rss link and generate link based on your current search -* improve view page * make renchon the favicon and official mascot * make sukebei db schema compatible with current code -* Site theme * comments in torrent view page +* Site theme * accounts? * scraping * Daily DB dumps From 8828a07fdff97950cf8d7068e8f780c08d7c3471 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 5 May 2017 15:55:25 +0200 Subject: [PATCH 11/11] Less autistic date display --- js/main.js | 40 ++++++++++++++++++++++++++++++++++++++-- templates/home.html | 2 +- templates/view.html | 2 +- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/js/main.js b/js/main.js index 04089014..2e9e87fa 100644 --- a/js/main.js +++ b/js/main.js @@ -22,12 +22,18 @@ var maxId = 5; for (var i = 0; i < maxId; i++) { var el = document.getElementById('page-' + i), n = prev + i; + if (el == null) + continue; el.href = pageString + n + query; el.innerHTML = n; } - document.getElementById('page-next').href = pageString + next + query; - document.getElementById('page-prev').href = pageString + prev + query; + var e = document.getElementById('page-next'); + if (e != null) + e.href = pageString + next + query; + var e = document.getElementById('page-prev'); + if (e != null) + e.href = pageString + prev + query; // Used by spoiler tags function toggleLayer(elem) { @@ -36,3 +42,33 @@ function toggleLayer(elem) { else elem.classList.add("hide"); } + +function formatDate(date) { // thanks stackoverflow + var monthNames = [ + "January", "February", "March", + "April", "May", "June", "July", + "August", "September", "October", + "November", "December" + ]; + + var day = date.getDate(); + var monthIndex = date.getMonth(); + var year = date.getFullYear(); + + return day + ' ' + monthNames[monthIndex] + ' ' + year; +} + +var list = document.getElementsByClassName("date-short"); +for(var i in list) { + var e = list[i]; + e.title = e.innerText; + e.innerText = formatDate(new Date(e.innerText)); +} + +var list = document.getElementsByClassName("date-full"); +for(var i in list) { + var e = list[i]; + e.title = e.innerText; + var date = new Date(e.innerText); + e.innerText = date.toDateString() + " " + date.toLocaleTimeString(); +} diff --git a/templates/home.html b/templates/home.html index ae84b500..a64d064c 100644 --- a/templates/home.html +++ b/templates/home.html @@ -31,7 +31,7 @@ {{.Name}} - {{.Date}} + {{.Date}} {{.Filesize}} diff --git a/templates/view.html b/templates/view.html index b113c36b..1f05062e 100644 --- a/templates/view.html +++ b/templates/view.html @@ -19,7 +19,7 @@ Date - {{.Date}} + {{.Date}} FileSize