Trackers in Torrents + Missing comments + Function renaming (#768)

* Missing comments and Function renaming

* Added some missing comments
* Renamed functions to get user followers/following
* GetFollowers to get followers
* GetLikings to get who the user is following

* Renaming + Add support of previous trackers

* Renaming user.Likings in user.Followers
* Renaming user.Liked in user.Likings
* Add a new string field Trackers in torrent model
* Trackers from torrent file are now populated to the databse
* Needed trackers are added to the torrent trackers if not provided or
if trackers is empty in DB (backward compatibility)

* New check and url encoding

* No more regex for verifying tracker url
* Encodes tracker url for "&" & "?" character possibly existing in
tracker url and breaking magnet link

* Improvements

* Trackers are now encoded in torrent.ParseTrackers
* Faster check by using the for loop of checktrackers
* No more boolean, we need to check len of array returned
* torrent.Trackers can be directly used in url as they are encoded like
: tr=tracker1&tr=tracker2&tr=...

config/navigation.go

@@ -6,5 +6,6 @@ package config
 const (
 	// TorrentsPerPage : Number of torrents per page
 	TorrentsPerPage = 50
+	// MaxTorrentsPerPage : maximum torrents per page
 	MaxTorrentsPerPage = 300
 )

config/search.go

@@ -9,7 +9,10 @@ type SearchConfig struct {
 var DefaultSearchConfig = SearchConfig{}
 
 const (
+	// DefaultElasticsearchAnalyzer : default analyzer for ES
 	DefaultElasticsearchAnalyzer = "nyaapantsu_analyzer"
+	// DefaultElasticsearchIndex : default search index for ES
 	DefaultElasticsearchIndex = "nyaapantsu"
-	DefaultElasticsearchType = "torrents" // Name of the type in the es mapping
+	// DefaultElasticsearchType :  Name of the type in the es mapping
+	DefaultElasticsearchType = "torrents"
 )

config/trackers.go

@@ -13,3 +13,8 @@ var Trackers = []string{
 	"udp://tracker.internetwarriors.net:1337/announce",
 	"http://mgtracker.org:6969/announce",
 	"http://tracker.baka-sub.cf/announce"}
+
+// NeededTrackers : Array indexes of Trackers for needed tracker in a torrent file
+var NeededTrackers = []int{
+	0,
+}

model/torrent.go

@@ -1,15 +1,19 @@
 package model
 
 import (
+	"context"
 	"fmt"
 	"html/template"
-	"context"
 	"path/filepath"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
+
 	elastic "gopkg.in/olivere/elastic.v5"
 
+	"net/url"
+
 	"github.com/NyaaPantsu/nyaa/config"
 	"github.com/NyaaPantsu/nyaa/util"
 	"github.com/bradfitz/slice"
@@ -52,6 +56,7 @@ type Torrent struct {
 	Filesize    int64     `gorm:"column:filesize"`
 	Description string    `gorm:"column:description"`
 	WebsiteLink string    `gorm:"column:website_link"`
+	Trackers    string    `gorm:"column:trackers"`
 	DeletedAt   *time.Time
 
 	Uploader    *User        `gorm:"AssociationForeignKey:UploaderID;ForeignKey:user_id"`
@@ -67,27 +72,9 @@ type Torrent struct {
 }
 
 // Size : Returns the total size of memory recursively allocated for this struct
-// FIXME: doesn't go have sizeof or something nicer for this?
+// FIXME: Is it deprecated?
 func (t Torrent) Size() (s int) {
-	s += 8 + // ints
-		2*3 + // time.Time
-		2 + // pointers
-		4*2 + // string pointers
-		// string array sizes
-		len(t.Name) + len(t.Hash) + len(t.Description) + len(t.WebsiteLink) +
-		2*2 // array pointers
-	s *= 8 // Assume 64 bit OS
-
-	if t.Uploader != nil {
-		s += t.Uploader.Size()
-	}
-	for _, c := range t.OldComments {
-		s += c.Size()
-	}
-	for _, c := range t.Comments {
-		s += c.Size()
-	}
-
+	s = int(reflect.TypeOf(t).Size())
 	return
 
 }
@@ -132,19 +119,21 @@ func (t *Torrent) IsDeleted() bool {
 	return t.DeletedAt != nil
 }
 
+// AddToESIndex : Adds a torrent to Elastic Search
 func (t Torrent) AddToESIndex(client *elastic.Client) error {
 	ctx := context.Background()
-	torrentJson := t.ToJSON()
+	torrentJSON := t.ToJSON()
 	_, err := client.Index().
 		Index(config.DefaultElasticsearchIndex).
 		Type(config.DefaultElasticsearchType).
-		Id(torrentJson.ID).
-		BodyJson(torrentJson).
+		Id(torrentJSON.ID).
+		BodyJson(torrentJSON).
 		Refresh("true").
 		Do(ctx)
 	return err
 }
 
+// DeleteFromESIndex : Removes a torrent from Elastic Search
 func (t Torrent) DeleteFromESIndex(client *elastic.Client) error {
 	ctx := context.Background()
 	_, err := client.Delete().
@@ -155,6 +144,38 @@ func (t Torrent) DeleteFromESIndex(client *elastic.Client) error {
 	return err
 }
 
+// ParseTrackers : Takes an array of trackers, adds needed trackers and parse it to url string
+func (t *Torrent) ParseTrackers(trackers []string) {
+	v := url.Values{}
+	if len(config.NeededTrackers) > 0 { // if we have some needed trackers configured
+		if len(trackers) == 0 {
+			trackers = config.Trackers
+		} else {
+			for _, id := range config.NeededTrackers {
+				found := false
+				for _, tracker := range trackers {
+					if tracker == config.Trackers[id] {
+						found = true
+						break
+					}
+				}
+				if !found {
+					trackers = append(trackers, config.Trackers[id])
+				}
+			}
+		}
+	}
+	v["tr"] = trackers
+	t.Trackers = v.Encode()
+}
+
+// GetTrackersArray : Convert trackers string to Array
+func (t *Torrent) GetTrackersArray() (trackers []string) {
+	v, _ := url.ParseQuery(t.Trackers)
+	trackers = v["tr"]
+	return
+}
+
 /* We need a JSON object instead of a Gorm structure because magnet URLs are
    not in the database and have to be generated dynamically */
 
@@ -208,7 +229,13 @@ type TorrentJSON struct {
 
 // ToJSON converts a model.Torrent to its equivalent JSON structure
 func (t *Torrent) ToJSON() TorrentJSON {
-	magnet := util.InfoHashToMagnet(strings.TrimSpace(t.Hash), t.Name, config.Trackers...)
+	var trackers []string
+	if t.Trackers == "" {
+		trackers = config.Trackers
+	} else {
+		trackers = t.GetTrackersArray()
+	}
+	magnet := util.InfoHashToMagnet(strings.TrimSpace(t.Hash), t.Name, trackers...)
 	commentsJSON := make([]CommentJSON, 0, len(t.OldComments)+len(t.Comments))
 	for _, c := range t.OldComments {
 		commentsJSON = append(commentsJSON, CommentJSON{Username: c.Username, UserID: -1, Content: template.HTML(c.Content), Date: c.Date.UTC()})

model/user.go

@@ -34,8 +34,8 @@ type User struct {
 	UserSettings   string    `gorm:"column:settings"`
 
 	// TODO: move this to PublicUser
-	Likings []User // Don't work `gorm:"foreignkey:user_id;associationforeignkey:follower_id;many2many:user_follows"`
-	Liked   []User // Don't work `gorm:"foreignkey:follower_id;associationforeignkey:user_id;many2many:user_follows"`
+	Followers []User // Don't work `gorm:"foreignkey:user_id;associationforeignkey:follower_id;many2many:user_follows"`
+	Likings   []User // Don't work `gorm:"foreignkey:follower_id;associationforeignkey:user_id;many2many:user_follows"`
 
 	MD5           string         `json:"md5" gorm:"column:md5"` // Hash of email address, used for Gravatar
 	Torrents      []Torrent      `gorm:"ForeignKey:UploaderID"`
@@ -137,8 +137,8 @@ func (u *User) ToJSON() UserJSON {
 		Username:    u.Username,
 		Status:      u.Status,
 		CreatedAt:   u.CreatedAt.Format(time.RFC3339),
-		LikingCount: len(u.Likings),
-		LikedCount:  len(u.Liked),
+		LikingCount: len(u.Followers),
+		LikedCount:  len(u.Likings),
 	}
 	return json
 }

router/upload.go

@@ -47,6 +47,7 @@ type uploadForm struct {
 	Filesize      int64
 	Filepath      string
 	FileList      []uploadedFile
+	Trackers      []string
 }
 
 // TODO: these should be in another package (?)
@@ -153,7 +154,8 @@ func (f *uploadForm) ExtractInfo(r *http.Request) error {
 			return errPrivateTorrent
 		}
 		trackers := torrent.GetAllAnnounceURLS()
-		if !uploadService.CheckTrackers(trackers) {
+		f.Trackers = uploadService.CheckTrackers(trackers)
+		if len(f.Trackers) == 0 {
 			return errTrackerProblem
 		}
 
@@ -209,6 +211,7 @@ func (f *uploadForm) ExtractInfo(r *http.Request) error {
 				return errors.New("Incorrect hash")
 			}
 		}
+		// TODO: Get Trackers from magnet URL
 		f.Filesize = 0
 		f.Filepath = ""
 	}

router/upload_handler.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"strconv"
 	"time"
+
 	elastic "gopkg.in/olivere/elastic.v5"
 
 	"github.com/NyaaPantsu/nyaa/config"
@@ -85,8 +86,7 @@ func UploadPostHandler(w http.ResponseWriter, r *http.Request) {
 			Description: uploadForm.Description,
 			WebsiteLink: uploadForm.WebsiteLink,
 			UploaderID:  user.ID}
-
-
+		torrent.ParseTrackers(uploadForm.Trackers)
 		db.ORM.Create(&torrent)
 
 		client, err := elastic.NewClient()
@@ -101,13 +101,12 @@ func UploadPostHandler(w http.ResponseWriter, r *http.Request) {
 			log.Errorf("Unable to create elasticsearch client: %s", err)
 		}
 
-
 		url, err := Router.Get("view_torrent").URL("id", strconv.FormatUint(uint64(torrent.ID), 10))
 
 		if user.ID > 0 && config.DefaultUserSettings["new_torrent"] { // If we are a member and notifications for new torrents are enabled
-			userService.GetLikings(user) // We populate the liked field for users
-			if len(user.Likings) > 0 {   // If we are followed by at least someone
-				for _, follower := range user.Likings {
+			userService.GetFollowers(user) // We populate the liked field for users
+			if len(user.Followers) > 0 {   // If we are followed by at least someone
+				for _, follower := range user.Followers {
 					follower.ParseSettings() // We need to call it before checking settings
 					if follower.Settings.Get("new_torrent") {
 						T, _, _ := languages.TfuncAndLanguageWithFallback(follower.Language, follower.Language) // We need to send the notification to every user in their language

service/api/api.go

@@ -176,7 +176,8 @@ func (r *TorrentRequest) ValidateMultipartUpload(req *http.Request) (int64, erro
 			return 0, errors.New("private torrents not allowed"), http.StatusNotAcceptable
 		}
 		trackers := torrent.GetAllAnnounceURLS()
-		if !uploadService.CheckTrackers(trackers) {
+		trackers = uploadService.CheckTrackers(trackers)
+		if len(trackers) == 0 {
 			return 0, errors.New("tracker(s) not allowed"), http.StatusNotAcceptable
 		}
 		if r.Name == "" {

service/upload/upload.go

@@ -3,12 +3,14 @@ package uploadService
 import (
 	"strings"
 
+	"net/url"
+
 	"github.com/NyaaPantsu/nyaa/config"
 	"github.com/NyaaPantsu/nyaa/model"
 )
 
 // CheckTrackers : Check if there is good trackers in torrent
-func CheckTrackers(trackers []string) bool {
+func CheckTrackers(trackers []string) []string {
 	// TODO: move to runtime configuration
 	var deadTrackers = []string{ // substring matches!
 		"://open.nyaatorrents.info:6544",
@@ -26,19 +28,23 @@ func CheckTrackers(trackers []string) bool {
 		"://tracker.prq.to",
 		"://bt.rghost.net"}
 
-	var numGood int
+	var trackerRet []string
 	for _, t := range trackers {
-		good := true
-		for _, check := range deadTrackers {
-			if strings.Contains(t, check) {
-				good = false
+		urlTracker, err := url.Parse(t)
+		if err == nil {
+			good := true
+			for _, check := range deadTrackers {
+				if strings.Contains(t, check) {
+					good = false
+					break // No need to continue the for loop
+				}
+			}
+			if good {
+				trackerRet = append(trackerRet, urlTracker.String())
 			}
 		}
-		if good {
-			numGood++
-		}
 	}
-	return numGood > 0
+	return trackerRet
 }
 
 // IsUploadEnabled : Check if upload is enabled in config

service/user/user.go

@@ -309,8 +309,8 @@ func RetrieveUserForAdmin(id string) (model.User, int, error) {
 	var liked, likings []model.User
 	db.ORM.Joins("JOIN user_follows on user_follows.user_id=?", user.ID).Where("users.user_id = user_follows.following").Group("users.user_id").Find(&likings)
 	db.ORM.Joins("JOIN user_follows on user_follows.following=?", user.ID).Where("users.user_id = user_follows.user_id").Group("users.user_id").Find(&liked)
-	user.Likings = likings
-	user.Liked = liked
+	user.Followers = likings
+	user.Likings = liked
 	return user, http.StatusOK, nil
 }
 
@@ -323,19 +323,19 @@ func RetrieveUsersForAdmin(limit int, offset int) ([]model.User, int) {
 	return users, nbUsers
 }
 
-// GetLiked : Gets who is following the user
-func GetLiked(user *model.User) *model.User {
+// GetLikings : Gets who is followed by the user
+func GetLikings(user *model.User) *model.User {
 	var liked []model.User
 	db.ORM.Joins("JOIN user_follows on user_follows.following=?", user.ID).Where("users.user_id = user_follows.user_id").Group("users.user_id").Find(&liked)
-	user.Liked = liked
+	user.Likings = liked
 	return user
 }
 
-// GetLikings : Gets who is followed by the user
-func GetLikings(user *model.User) *model.User {
+// GetFollowers : Gets who is following the user
+func GetFollowers(user *model.User) *model.User {
 	var likings []model.User
 	db.ORM.Joins("JOIN user_follows on user_follows.user_id=?", user.ID).Where("users.user_id = user_follows.following").Group("users.user_id").Find(&likings)
-	user.Likings = likings
+	user.Followers = likings
 	return user
 }