From 2da2ad421424033e58d0e6edc4e4eda1674f3ec2 Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 08:20:34 -0300 Subject: [PATCH 01/11] FilesizeFetcher -> MetainfoFetcher --- config/config.go | 6 ++--- ...{filesizeFetcher.go => metainfoFetcher.go} | 4 +-- main.go | 12 ++++----- .../metainfoFetcher.go} | 26 +++++++++---------- .../operation.go | 6 ++--- .../operation_test.go | 4 +-- 6 files changed, 29 insertions(+), 29 deletions(-) rename config/{filesizeFetcher.go => metainfoFetcher.go} (74%) rename service/torrent/{filesizeFetcher/filesizeFetcher.go => metainfoFetcher/metainfoFetcher.go} (88%) rename service/torrent/{filesizeFetcher => metainfoFetcher}/operation.go (91%) rename service/torrent/{filesizeFetcher => metainfoFetcher}/operation_test.go (93%) diff --git a/config/config.go b/config/config.go index a47a4a93..be354d2e 100644 --- a/config/config.go +++ b/config/config.go @@ -32,12 +32,12 @@ type Config struct { // optional i2p configuration I2P *I2PConfig `json:"i2p"` // filesize fetcher config - FilesizeFetcher FilesizeFetcherConfig `json:"filesize_fetcher"` + MetainfoFetcher MetainfoFetcherConfig `json:"metainfo_fetcher"` // internationalization config I18n I18nConfig `json:"i18n"` } -var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", "default", DefaultScraperConfig, DefaultCacheConfig, DefaultSearchConfig, nil, DefaultFilesizeFetcherConfig, DefaultI18nConfig} +var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", "default", DefaultScraperConfig, DefaultCacheConfig, DefaultSearchConfig, nil, DefaultMetainfoFetcherConfig, DefaultI18nConfig} var allowedDatabaseTypes = map[string]bool{ @@ -62,7 +62,7 @@ func New() *Config { config.DBLogMode = Defaults.DBLogMode config.Scrape = Defaults.Scrape config.Cache = Defaults.Cache - config.FilesizeFetcher = Defaults.FilesizeFetcher + config.MetainfoFetcher = Defaults.MetainfoFetcher config.I18n = Defaults.I18n return &config } diff --git a/config/filesizeFetcher.go b/config/metainfoFetcher.go similarity index 74% rename from config/filesizeFetcher.go rename to config/metainfoFetcher.go index c86082fb..792642f8 100644 --- a/config/filesizeFetcher.go +++ b/config/metainfoFetcher.go @@ -1,13 +1,13 @@ package config -type FilesizeFetcherConfig struct { +type MetainfoFetcherConfig struct { QueueSize int `json:"queue_size"` Timeout int `json:"timeout"` MaxDays int `json:"max_days"` WakeUpInterval int `json:"wake_up_interval"` } -var DefaultFilesizeFetcherConfig = FilesizeFetcherConfig{ +var DefaultMetainfoFetcherConfig = MetainfoFetcherConfig{ QueueSize: 10, Timeout: 120, // 2 min MaxDays: 90, diff --git a/main.go b/main.go index e15ba6de..a22da754 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( "github.com/ewhal/nyaa/network" "github.com/ewhal/nyaa/router" "github.com/ewhal/nyaa/service/scraper" - "github.com/ewhal/nyaa/service/torrent/filesizeFetcher" + "github.com/ewhal/nyaa/service/torrent/metainfoFetcher" "github.com/ewhal/nyaa/util/languages" "github.com/ewhal/nyaa/util/log" "github.com/ewhal/nyaa/util/search" @@ -83,9 +83,9 @@ func RunScraper(conf *config.Config) { scraper.Wait() } -// RunFilesizeFetcher runs the database filesize fetcher main loop -func RunFilesizeFetcher(conf *config.Config) { - fetcher, err := filesizeFetcher.New(&conf.FilesizeFetcher) +// RunMetainfoFetcher runs the database filesize fetcher main loop +func RunMetainfoFetcher(conf *config.Config) { + fetcher, err := metainfoFetcher.New(&conf.MetainfoFetcher) if err != nil { log.Fatalf("failed to start fetcher, %s", err) return @@ -147,8 +147,8 @@ func main() { RunScraper(conf) } else if *mode == "webapp" { RunServer(conf) - } else if *mode == "filesize_fetcher" { - RunFilesizeFetcher(conf) + } else if *mode == "metainfo_fetcher" { + RunMetainfoFetcher(conf) } else { log.Fatalf("invalid runtime mode: %s", *mode) } diff --git a/service/torrent/filesizeFetcher/filesizeFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go similarity index 88% rename from service/torrent/filesizeFetcher/filesizeFetcher.go rename to service/torrent/metainfoFetcher/metainfoFetcher.go index 1f56f0c3..088606ba 100644 --- a/service/torrent/filesizeFetcher/filesizeFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -1,4 +1,4 @@ -package filesizeFetcher; +package metainfoFetcher; import ( "github.com/anacrolix/torrent" @@ -13,7 +13,7 @@ import ( "time" ) -type FilesizeFetcher struct { +type MetainfoFetcher struct { torrentClient *torrent.Client results chan Result queueSize int @@ -27,9 +27,9 @@ type FilesizeFetcher struct { wg sync.WaitGroup } -func New(fetcherConfig *config.FilesizeFetcherConfig) (fetcher *FilesizeFetcher, err error) { +func New(fetcherConfig *config.MetainfoFetcherConfig) (fetcher *MetainfoFetcher, err error) { client, err := torrent.NewClient(nil) - fetcher = &FilesizeFetcher{ + fetcher = &MetainfoFetcher{ torrentClient: client, results: make(chan Result, fetcherConfig.QueueSize), queueSize: fetcherConfig.QueueSize, @@ -43,7 +43,7 @@ func New(fetcherConfig *config.FilesizeFetcherConfig) (fetcher *FilesizeFetcher, return } -func (fetcher *FilesizeFetcher) isFetchingOrFailed(t model.Torrent) bool { +func (fetcher *MetainfoFetcher) isFetchingOrFailed(t model.Torrent) bool { for _, op := range fetcher.queue { if op.torrent.ID == t.ID { return true @@ -54,7 +54,7 @@ func (fetcher *FilesizeFetcher) isFetchingOrFailed(t model.Torrent) bool { return ok } -func (fetcher *FilesizeFetcher) addToQueue(op *FetchOperation) bool { +func (fetcher *MetainfoFetcher) addToQueue(op *FetchOperation) bool { fetcher.queueMutex.Lock() defer fetcher.queueMutex.Unlock() @@ -67,7 +67,7 @@ func (fetcher *FilesizeFetcher) addToQueue(op *FetchOperation) bool { } -func (fetcher *FilesizeFetcher) removeFromQueue(op *FetchOperation) bool { +func (fetcher *MetainfoFetcher) removeFromQueue(op *FetchOperation) bool { fetcher.queueMutex.Lock() defer fetcher.queueMutex.Unlock() @@ -111,7 +111,7 @@ func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error { return nil } -func (fetcher *FilesizeFetcher) gotResult(r Result) { +func (fetcher *MetainfoFetcher) gotResult(r Result) { updatedSuccessfully := false if r.err != nil { log.Infof("Failed to get torrent filesize (TID: %d), err %v", r.operation.torrent.ID, r.err) @@ -141,7 +141,7 @@ func (fetcher *FilesizeFetcher) gotResult(r Result) { fetcher.removeFromQueue(r.operation) } -func (fetcher *FilesizeFetcher) fillQueue() { +func (fetcher *MetainfoFetcher) fillQueue() { toFill := fetcher.queueSize - len(fetcher.queue) if toFill <= 0 { @@ -180,7 +180,7 @@ func (fetcher *FilesizeFetcher) fillQueue() { } } -func (fetcher *FilesizeFetcher) run() { +func (fetcher *MetainfoFetcher) run() { var result Result defer fetcher.wg.Done() @@ -202,13 +202,13 @@ func (fetcher *FilesizeFetcher) run() { } } -func (fetcher *FilesizeFetcher) RunAsync() { +func (fetcher *MetainfoFetcher) RunAsync() { fetcher.wg.Add(1) go fetcher.run() } -func (fetcher *FilesizeFetcher) Close() error { +func (fetcher *MetainfoFetcher) Close() error { fetcher.queueMutex.Lock() defer fetcher.queueMutex.Unlock() @@ -222,7 +222,7 @@ func (fetcher *FilesizeFetcher) Close() error { return nil } -func (fetcher *FilesizeFetcher) Wait() { +func (fetcher *MetainfoFetcher) Wait() { fetcher.wg.Wait() } diff --git a/service/torrent/filesizeFetcher/operation.go b/service/torrent/metainfoFetcher/operation.go similarity index 91% rename from service/torrent/filesizeFetcher/operation.go rename to service/torrent/metainfoFetcher/operation.go index 1a08d061..6c8e6905 100644 --- a/service/torrent/filesizeFetcher/operation.go +++ b/service/torrent/metainfoFetcher/operation.go @@ -1,4 +1,4 @@ -package filesizeFetcher; +package metainfoFetcher; import ( "github.com/anacrolix/torrent/metainfo" @@ -11,7 +11,7 @@ import ( ) type FetchOperation struct { - fetcher *FilesizeFetcher + fetcher *MetainfoFetcher torrent model.Torrent done chan int } @@ -22,7 +22,7 @@ type Result struct { info *metainfo.Info } -func NewFetchOperation(fetcher *FilesizeFetcher, dbEntry model.Torrent) (op *FetchOperation) { +func NewFetchOperation(fetcher *MetainfoFetcher, dbEntry model.Torrent) (op *FetchOperation) { op = &FetchOperation{ fetcher: fetcher, torrent: dbEntry, diff --git a/service/torrent/filesizeFetcher/operation_test.go b/service/torrent/metainfoFetcher/operation_test.go similarity index 93% rename from service/torrent/filesizeFetcher/operation_test.go rename to service/torrent/metainfoFetcher/operation_test.go index 21e36faa..1be73474 100644 --- a/service/torrent/filesizeFetcher/operation_test.go +++ b/service/torrent/metainfoFetcher/operation_test.go @@ -1,4 +1,4 @@ -package filesizeFetcher; +package metainfoFetcher; import ( "testing" @@ -13,7 +13,7 @@ func TestInvalidHash(t *testing.T) { t.Skipf("Failed to create client, with err %v. Skipping.", err) } - fetcher := &FilesizeFetcher{ + fetcher := &MetainfoFetcher{ timeout: 5, torrentClient: client, results: make(chan Result, 1), From e1f1c9d0c3821cea590a14bf3d6c206f10a4a042 Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 09:05:20 -0300 Subject: [PATCH 02/11] Update MetainfoFetcher query logic It used to update only torrents with NULL or 0 filesizes, setting both their filesizes and the file lists. Now, it looks for both NULL or 0 filesizes and empty file lists. --- .../metainfoFetcher/metainfoFetcher.go | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/service/torrent/metainfoFetcher/metainfoFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go index 088606ba..95e97103 100644 --- a/service/torrent/metainfoFetcher/metainfoFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -82,8 +82,9 @@ func (fetcher *MetainfoFetcher) removeFromQueue(op *FetchOperation) bool { } func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error { - log.Infof("TID %d has %d files.", dbEntry.ID, len(info.Files)) - for _, file := range info.Files { + torrentFiles := info.UpvertedFiles() + log.Infof("TID %d has %d files.", dbEntry.ID, len(torrentFiles)) + for _, file := range torrentFiles { path := file.DisplayPath(info) fileExists := false for _, existingFile := range dbEntry.FileList { @@ -114,7 +115,7 @@ func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error { func (fetcher *MetainfoFetcher) gotResult(r Result) { updatedSuccessfully := false if r.err != nil { - log.Infof("Failed to get torrent filesize (TID: %d), err %v", r.operation.torrent.ID, r.err) + log.Infof("Failed to get torrent metainfo (TID: %d), err %v", r.operation.torrent.ID, r.err) } else if r.info.TotalLength() == 0 { log.Infof("Got length 0 for torrent TID: %d. Possible bug?", r.operation.torrent.ID) } else { @@ -127,10 +128,12 @@ func (fetcher *MetainfoFetcher) gotResult(r Result) { updatedSuccessfully = true } - // Also update the File list with FileInfo, I guess. - err = updateFileList(r.operation.torrent, r.info) - if err != nil { - log.Infof("Failed to update file list of TID %d", r.operation.torrent.ID) + // Create the file list, if it's missing. + if len(r.operation.torrent.FileList) == 0 { + err = updateFileList(r.operation.torrent, r.info) + if err != nil { + log.Infof("Failed to update file list of TID %d", r.operation.torrent.ID) + } } } @@ -149,17 +152,19 @@ func (fetcher *MetainfoFetcher) fillQueue() { } oldest := time.Now().Add(0 - (time.Hour * time.Duration(24 * fetcher.maxDays))) - params := serviceBase.CreateWhereParams("(filesize IS NULL OR filesize = 0) AND date > ?", oldest) + // Nice query lol + // Select the torrents with no filesize, or without any rows with torrent_id in the files table, that are younger than fetcher.MaxDays + params := serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ?", oldest) // Get up to queueSize + len(failed) torrents, so we get at least some fresh new ones. dbTorrents, count, err := torrentService.GetTorrents(params, fetcher.queueSize + len(fetcher.failedOperations), 0) if err != nil { - log.Infof("Failed to get torrents for filesize updating") + log.Infof("Failed to get torrents for metainfo updating") return } if count == 0 { - log.Infof("No torrents for filesize update") + log.Infof("No torrents for metainfo update") return } From 839e6068df2b15cb0e6b68fd323e655c25793dee Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 10:19:19 -0300 Subject: [PATCH 03/11] Add FileList view to view.html template Also added the code for FileList loading on torrentService. --- db/gorm.go | 6 +++++- model/file.go | 6 ++---- model/torrent.go | 12 ++++++++---- service/torrent/torrent.go | 2 +- templates/view.html | 21 +++++++++++++++++++++ translations/en-us.all.json | 8 ++++++++ 6 files changed, 45 insertions(+), 10 deletions(-) diff --git a/db/gorm.go b/db/gorm.go index 88c7db2c..74c42a07 100644 --- a/db/gorm.go +++ b/db/gorm.go @@ -58,7 +58,11 @@ func GormInit(conf *config.Config, logger Logger) (*gorm.DB, error) { if db.Error != nil { return db, db.Error } - db.AutoMigrate(&model.Torrent{}, &model.TorrentReport{}, &model.File{}) + db.AutoMigrate(&model.Torrent{}, &model.TorrentReport{}) + if db.Error != nil { + return db, db.Error + } + db.AutoMigrate(&model.File{}) if db.Error != nil { return db, db.Error } diff --git a/model/file.go b/model/file.go index 4a95c6dc..3f27e944 100644 --- a/model/file.go +++ b/model/file.go @@ -4,13 +4,11 @@ type File struct { ID uint `gorm:"column:file_id;primary_key"` TorrentID uint `gorm:"column:torrent_id"` Path string `gorm:"column:path"` - Filesize int64 `gorm:"column:filesize"` - - Torrent *Torrent `gorm:"AssociationForeignKey:TorrentID;ForeignKey:torrent_id"` + Filesize int64 `gorm:"column:filesize"` } // Returns the total size of memory allocated for this struct func (f File) Size() int { - return (1 + len(f.Path) + 2) * 8; + return (2 + len(f.Path) + 2) * 8; } diff --git a/model/torrent.go b/model/torrent.go index 50e83031..de081167 100644 --- a/model/torrent.go +++ b/model/torrent.go @@ -90,8 +90,8 @@ type CommentJSON struct { } type FileJSON struct { - Path string `json:"path"` - Length int64 `json:"length"` + Path string `json:"path"` + Filesize string `json:"filesize"` } type TorrentJSON struct { @@ -116,7 +116,7 @@ type TorrentJSON struct { Leechers uint32 `json:"leechers"` Completed uint32 `json:"completed"` LastScrape time.Time `json:"last_scrape"` - FileList []File `json:"file_list"` + FileList []FileJSON `json:"file_list"` } // ToJSON converts a model.Torrent to its equivalent JSON structure @@ -129,6 +129,10 @@ func (t *Torrent) ToJSON() TorrentJSON { for _, c := range t.Comments { commentsJSON = append(commentsJSON, CommentJSON{Username: c.User.Username, UserID: int(c.User.ID), Content: util.MarkdownToHTML(c.Content), Date: c.CreatedAt.UTC()}) } + fileListJSON := make([]FileJSON, 0, len(t.FileList)) + for _, f := range t.FileList { + fileListJSON = append(fileListJSON, FileJSON{Path: f.Path, Filesize: util.FormatFilesize2(f.Filesize)}) + } uploader := "" if t.Uploader != nil { uploader = t.Uploader.Username @@ -162,7 +166,7 @@ func (t *Torrent) ToJSON() TorrentJSON { Seeders: t.Seeders, Completed: t.Completed, LastScrape: t.LastScrape, - FileList: t.FileList, + FileList: fileListJSON, } return res diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go index cadfff5f..65065991 100644 --- a/service/torrent/torrent.go +++ b/service/torrent/torrent.go @@ -53,7 +53,7 @@ func GetTorrentById(id string) (torrent model.Torrent, err error) { return } - tmp := db.ORM.Where("torrent_id = ?", id).Preload("Comments") + tmp := db.ORM.Where("torrent_id = ?", id).Preload("Comments").Preload("FileList") err = tmp.Error if err != nil { return diff --git a/templates/view.html b/templates/view.html index bfab7acb..0791c3eb 100644 --- a/templates/view.html +++ b/templates/view.html @@ -80,6 +80,27 @@
+ {{ if gt (len .FileList) 0 }} +
+
+

{{T "files"}}

+
+ + + + + + {{ range .FileList }} + + + + + {{ end }} +
{{T "filename"}}{{T "size"}}
{{.Path}}{{.Filesize}}
+
+
+
+ {{ end }}

{{T "comments"}}

diff --git a/translations/en-us.all.json b/translations/en-us.all.json index 3d913c46..8ccf7f87 100644 --- a/translations/en-us.all.json +++ b/translations/en-us.all.json @@ -646,5 +646,13 @@ { "id": "delete", "translation": "Delete" + }, + { + "id": "files", + "translation": "Files" + }, + { + "id": "filename", + "translation": "Filename" } ] From 7d8cbe1393c73db42697e18867276511c6dfb6f5 Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 11:35:03 -0300 Subject: [PATCH 04/11] Add Upload and Download limiters, rely on unique_index for avoiding duplicates The FileList that is read with GetTorrents is empty, as it's only loaded with GetTorrentById. So, always insert the new FileList on the database, and rely on the unique_index to avoid duplications. --- config/metainfoFetcher.go | 12 +++-- model/file.go | 4 +- .../metainfoFetcher/metainfoFetcher.go | 44 +++++++++++-------- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/config/metainfoFetcher.go b/config/metainfoFetcher.go index 792642f8..eceb37a1 100644 --- a/config/metainfoFetcher.go +++ b/config/metainfoFetcher.go @@ -5,12 +5,18 @@ type MetainfoFetcherConfig struct { Timeout int `json:"timeout"` MaxDays int `json:"max_days"` WakeUpInterval int `json:"wake_up_interval"` + + UploadRateLimiter int `json:"upload_rate_limiter"` + DownloadRateLimiter int `json:"download_rate_limiter"` } var DefaultMetainfoFetcherConfig = MetainfoFetcherConfig{ - QueueSize: 10, - Timeout: 120, // 2 min - MaxDays: 90, + QueueSize: 10, + Timeout: 120, // 2 min + MaxDays: 90, WakeUpInterval: 300, // 5 min + + UploadRateLimiter: 1024, + DownloadRateLimiter: 1024, } diff --git a/model/file.go b/model/file.go index 3f27e944..67b64168 100644 --- a/model/file.go +++ b/model/file.go @@ -2,8 +2,8 @@ package model type File struct { ID uint `gorm:"column:file_id;primary_key"` - TorrentID uint `gorm:"column:torrent_id"` - Path string `gorm:"column:path"` + TorrentID uint `gorm:"column:torrent_id;unique_index:idx_tid_path"` + Path string `gorm:"column:path;unique_index:idx_tid_path"` Filesize int64 `gorm:"column:filesize"` } diff --git a/service/torrent/metainfoFetcher/metainfoFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go index 95e97103..e05263e5 100644 --- a/service/torrent/metainfoFetcher/metainfoFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -9,6 +9,7 @@ import ( "github.com/ewhal/nyaa/util/log" serviceBase "github.com/ewhal/nyaa/service" torrentService "github.com/ewhal/nyaa/service/torrent" + "golang.org/x/time/rate" "sync" "time" ) @@ -28,7 +29,18 @@ type MetainfoFetcher struct { } func New(fetcherConfig *config.MetainfoFetcherConfig) (fetcher *MetainfoFetcher, err error) { - client, err := torrent.NewClient(nil) + clientConfig := torrent.Config{} + // Well, it seems this is the right way to convert speed -> rate.Limiter + // https://github.com/anacrolix/torrent/blob/master/cmd/torrent/main.go + if fetcherConfig.UploadRateLimiter != -1 { + clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.UploadRateLimiter * 1024), 256<<10) + } + if fetcherConfig.DownloadRateLimiter != -1 { + clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.DownloadRateLimiter * 1024), 1<<20) + } + + client, err := torrent.NewClient(&clientConfig) + fetcher = &MetainfoFetcher{ torrentClient: client, results: make(chan Result, fetcherConfig.QueueSize), @@ -86,26 +98,19 @@ func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error { log.Infof("TID %d has %d files.", dbEntry.ID, len(torrentFiles)) for _, file := range torrentFiles { path := file.DisplayPath(info) - fileExists := false - for _, existingFile := range dbEntry.FileList { - if existingFile.Path == path { - fileExists = true - break - } + + // Can't read FileList from the GetTorrents output, rely on the unique_index + // to ensure no files are duplicated. + log.Infof("Adding file %s to filelist of TID %d", path, dbEntry.ID) + dbFile := model.File{ + TorrentID: dbEntry.ID, + Path: path, + Filesize: file.Length, } - if !fileExists { - log.Infof("Adding file %s to filelist of TID %d", path, dbEntry.ID) - dbFile := model.File{ - TorrentID: dbEntry.ID, - Path: path, - Filesize: file.Length, - } - - err := db.ORM.Create(&dbFile).Error - if err != nil { - return err - } + err := db.ORM.Create(&dbFile).Error + if err != nil { + return err } } @@ -223,6 +228,7 @@ func (fetcher *MetainfoFetcher) Close() error { } fetcher.done <- 1 + log.Infof("Send done signal to everyone, waiting...") fetcher.wg.Wait() return nil } From 58869114d7ddbc8aa5f008ff2ad36bbb385fa71d Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 11:47:13 -0300 Subject: [PATCH 05/11] Don't update Length if it hasn't changed --- .../metainfoFetcher/metainfoFetcher.go | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/service/torrent/metainfoFetcher/metainfoFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go index e05263e5..30f289da 100644 --- a/service/torrent/metainfoFetcher/metainfoFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -124,22 +124,29 @@ func (fetcher *MetainfoFetcher) gotResult(r Result) { } else if r.info.TotalLength() == 0 { log.Infof("Got length 0 for torrent TID: %d. Possible bug?", r.operation.torrent.ID) } else { - log.Infof("Got length %d for torrent TID: %d. Updating.", r.info.TotalLength(), r.operation.torrent.ID) - r.operation.torrent.Filesize = r.info.TotalLength() - _, err := torrentService.UpdateTorrent(r.operation.torrent) - if err != nil { - log.Infof("Failed to update torrent TID: %d with new filesize", r.operation.torrent.ID) - } else { - updatedSuccessfully = true - } + lengthOK := true - // Create the file list, if it's missing. - if len(r.operation.torrent.FileList) == 0 { - err = updateFileList(r.operation.torrent, r.info) + if r.operation.torrent.Filesize != r.info.TotalLength() { + log.Infof("Got length %d for torrent TID: %d. Updating.", r.info.TotalLength(), r.operation.torrent.ID) + r.operation.torrent.Filesize = r.info.TotalLength() + _, err := torrentService.UpdateTorrent(r.operation.torrent) if err != nil { - log.Infof("Failed to update file list of TID %d", r.operation.torrent.ID) + log.Infof("Failed to update torrent TID: %d with new filesize", r.operation.torrent.ID) + lengthOK = false } } + + filelistOK := true + // Create the file list, if it's missing. + if len(r.operation.torrent.FileList) == 0 { + err := updateFileList(r.operation.torrent, r.info) + if err != nil { + log.Infof("Failed to update file list of TID %d", r.operation.torrent.ID) + filelistOK = false + } + } + + updatedSuccessfully = lengthOK && filelistOK } if !updatedSuccessfully { From 720c27e38f1bb7b7c0ed70ca2815bc49532009c7 Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 12:28:48 -0300 Subject: [PATCH 06/11] Get FileList of new uploaded .torrent files --- router/upload.go | 17 +++++++++++++++++ router/uploadHandler.go | 12 ++++++++++++ 2 files changed, 29 insertions(+) diff --git a/router/upload.go b/router/upload.go index 2b143da9..57a65759 100644 --- a/router/upload.go +++ b/router/upload.go @@ -24,6 +24,13 @@ import ( "github.com/zeebo/bencode" ) +// Use this, because we seem to avoid using models, and we would need +// the torrent ID to create the File in the DB +type UploadedFile struct { + Path string + Filesize int64 +} + // UploadForm serializing HTTP form for torrent upload type UploadForm struct { Name string @@ -39,6 +46,7 @@ type UploadForm struct { SubCategoryID int Filesize int64 Filepath string + FileList []UploadedFile } // TODO: these should be in another package (?) @@ -150,6 +158,15 @@ func (f *UploadForm) ExtractInfo(r *http.Request) error { // extract filesize f.Filesize = int64(torrent.TotalSize()) + + // extract filelist + fileInfos := torrent.Info.GetFiles() + for _, info := range fileInfos { + f.FileList = append(f.FileList, UploadedFile{ + Path: info.Path.FilePath(), + Filesize: int64(info.Length), + }) + } } else { // No torrent file provided magnetUrl, err := url.Parse(string(f.Magnet)) //? diff --git a/router/uploadHandler.go b/router/uploadHandler.go index 50bb47da..e395889d 100644 --- a/router/uploadHandler.go +++ b/router/uploadHandler.go @@ -60,6 +60,18 @@ func UploadHandler(w http.ResponseWriter, r *http.Request) { Description: uploadForm.Description, UploaderID: user.ID} db.ORM.Create(&torrent) + + // add filelist to files db, if we have one + if len(uploadForm.FileList) > 0 { + for _, uploadedFile := range uploadForm.FileList { + file := model.File{ + TorrentID: torrent.ID, + Path: uploadedFile.Path, + Filesize: uploadedFile.Filesize} + db.ORM.Create(&file) + } + } + url, err := Router.Get("view_torrent").URL("id", strconv.FormatUint(uint64(torrent.ID), 10)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) From 83de593a65cf8548b49b75ff8d9a1684340aa814 Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 16:47:48 -0300 Subject: [PATCH 07/11] Exclude failed operations from query Instead of loading more rows to compensate for failed operations, exclude them at the query. --- .../torrent/metainfoFetcher/metainfoFetcher.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/service/torrent/metainfoFetcher/metainfoFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go index 30f289da..d84b9883 100644 --- a/service/torrent/metainfoFetcher/metainfoFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -164,11 +164,21 @@ func (fetcher *MetainfoFetcher) fillQueue() { } oldest := time.Now().Add(0 - (time.Hour * time.Duration(24 * fetcher.maxDays))) + excludedIDS := make([]uint, 0, len(fetcher.failedOperations)) + for id, _ := range fetcher.failedOperations { + excludedIDS = append(excludedIDS, id) + } + // Nice query lol // Select the torrents with no filesize, or without any rows with torrent_id in the files table, that are younger than fetcher.MaxDays - params := serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ?", oldest) - // Get up to queueSize + len(failed) torrents, so we get at least some fresh new ones. - dbTorrents, count, err := torrentService.GetTorrents(params, fetcher.queueSize + len(fetcher.failedOperations), 0) + var params serviceBase.WhereParams + + if len(excludedIDS) > 0 { + params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ? AND torrent_id NOT IN (?)", oldest, excludedIDS) + } else { + params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ?", oldest) + } + dbTorrents, count, err := torrentService.GetTorrents(params, fetcher.queueSize, 0) if err != nil { log.Infof("Failed to get torrents for metainfo updating") From cae0026a674d30c70f0e0b64ad4c117f2284aa01 Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 18:10:39 -0300 Subject: [PATCH 08/11] Add fail cooldown After a set cooldown, torrents that failed to be fetch will be able to be fetched again. Set FailCooldown to -1 to disable. --- config/metainfoFetcher.go | 4 +- .../metainfoFetcher/metainfoFetcher.go | 40 ++++++++++++++----- service/torrent/metainfoFetcher/operation.go | 10 ++--- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/config/metainfoFetcher.go b/config/metainfoFetcher.go index eceb37a1..8afe58a5 100644 --- a/config/metainfoFetcher.go +++ b/config/metainfoFetcher.go @@ -4,6 +4,7 @@ type MetainfoFetcherConfig struct { QueueSize int `json:"queue_size"` Timeout int `json:"timeout"` MaxDays int `json:"max_days"` + FailCooldown int `json:"fail_cooldown"` WakeUpInterval int `json:"wake_up_interval"` UploadRateLimiter int `json:"upload_rate_limiter"` @@ -14,9 +15,10 @@ var DefaultMetainfoFetcherConfig = MetainfoFetcherConfig{ QueueSize: 10, Timeout: 120, // 2 min MaxDays: 90, + FailCooldown: 30 * 60, // in seconds, when failed torrents will be able to be fetched again. WakeUpInterval: 300, // 5 min - UploadRateLimiter: 1024, + UploadRateLimiter: 1024, // kbps DownloadRateLimiter: 1024, } diff --git a/service/torrent/metainfoFetcher/metainfoFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go index d84b9883..807bc381 100644 --- a/service/torrent/metainfoFetcher/metainfoFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -20,10 +20,11 @@ type MetainfoFetcher struct { queueSize int timeout int maxDays int + failCooldown int done chan int queue []*FetchOperation queueMutex sync.Mutex - failedOperations map[uint]struct{} + failedOperations map[uint]time.Time wakeUp *time.Ticker wg sync.WaitGroup } @@ -47,8 +48,9 @@ func New(fetcherConfig *config.MetainfoFetcherConfig) (fetcher *MetainfoFetcher, queueSize: fetcherConfig.QueueSize, timeout: fetcherConfig.Timeout, maxDays: fetcherConfig.MaxDays, + failCooldown: fetcherConfig.FailCooldown, done: make(chan int, 1), - failedOperations: make(map[uint]struct{}), + failedOperations: make(map[uint]time.Time), wakeUp: time.NewTicker(time.Second * time.Duration(fetcherConfig.WakeUpInterval)), } @@ -150,12 +152,28 @@ func (fetcher *MetainfoFetcher) gotResult(r Result) { } if !updatedSuccessfully { - fetcher.failedOperations[r.operation.torrent.ID] = struct{}{} + fetcher.failedOperations[r.operation.torrent.ID] = time.Now() } fetcher.removeFromQueue(r.operation) } +func (fetcher *MetainfoFetcher) removeOldFailures() { + // Cooldown is disabled + if fetcher.failCooldown < 0 { + return + } + + now := time.Now() + for id, failTime := range fetcher.failedOperations { + if failTime.Add(time.Duration(fetcher.failCooldown) * time.Second).Before(now) { + log.Infof("Torrent TID %d gone through cooldown, removing from failures") + // Deleting keys inside a loop seems to be safe. + delete(fetcher.failedOperations, id) + } + } +} + func (fetcher *MetainfoFetcher) fillQueue() { toFill := fetcher.queueSize - len(fetcher.queue) @@ -180,15 +198,16 @@ func (fetcher *MetainfoFetcher) fillQueue() { } dbTorrents, count, err := torrentService.GetTorrents(params, fetcher.queueSize, 0) + if count == 0 { + log.Infof("No torrents for filesize update") + return + } + if err != nil { log.Infof("Failed to get torrents for metainfo updating") return } - - if count == 0 { - log.Infof("No torrents for metainfo update") - return - } + for _, T := range dbTorrents { if fetcher.isFetchingOrFailed(T) { @@ -215,15 +234,15 @@ func (fetcher *MetainfoFetcher) run() { done := 0 fetcher.fillQueue() for done == 0 { + fetcher.removeOldFailures() + fetcher.fillQueue() select { case done = <-fetcher.done: break case result = <-fetcher.results: fetcher.gotResult(result) - fetcher.fillQueue() break case <-fetcher.wakeUp.C: - fetcher.fillQueue() break } } @@ -245,6 +264,7 @@ func (fetcher *MetainfoFetcher) Close() error { } fetcher.done <- 1 + fetcher.torrentClient.Close() log.Infof("Send done signal to everyone, waiting...") fetcher.wg.Wait() return nil diff --git a/service/torrent/metainfoFetcher/operation.go b/service/torrent/metainfoFetcher/operation.go index 6c8e6905..92eada2e 100644 --- a/service/torrent/metainfoFetcher/operation.go +++ b/service/torrent/metainfoFetcher/operation.go @@ -41,20 +41,20 @@ func (op *FetchOperation) Start(out chan Result) { out <- Result{op, err, nil} return } + defer downloadingTorrent.Drop() - timeoutTicker := time.NewTicker(time.Second * time.Duration(op.fetcher.timeout)) + timeoutTimer := time.NewTicker(time.Second * time.Duration(op.fetcher.timeout)) + defer timeoutTimer.Stop() select { case <-downloadingTorrent.GotInfo(): - downloadingTorrent.Drop() out <- Result{op, nil, downloadingTorrent.Info()} break - case <-timeoutTicker.C: - downloadingTorrent.Drop() + case <-timeoutTimer.C: out <- Result{op, errors.New("Timeout"), nil} break case <-op.done: - downloadingTorrent.Drop() break } } + From a55cf2a8031dad7aa5edcddfe5814d6e5971b081 Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 18:24:41 -0300 Subject: [PATCH 09/11] Use NoCount version of GetTorrents The normal GetTorrents version always called COUNT(*) on the query, which consistently took around 300 ms. With the NoCount, the SELECT queries take 5-50 ms. --- service/torrent/metainfoFetcher/metainfoFetcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/torrent/metainfoFetcher/metainfoFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go index 807bc381..179eca5f 100644 --- a/service/torrent/metainfoFetcher/metainfoFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -196,9 +196,9 @@ func (fetcher *MetainfoFetcher) fillQueue() { } else { params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ?", oldest) } - dbTorrents, count, err := torrentService.GetTorrents(params, fetcher.queueSize, 0) + dbTorrents, err := torrentService.GetTorrentsOrderByNoCount(¶ms, "", fetcher.queueSize, 0) - if count == 0 { + if len(dbTorrents) == 0 { log.Infof("No torrents for filesize update") return } From 3dced6fdf053f9236f090906c4ab9ac801bf5638 Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 19:30:56 -0300 Subject: [PATCH 10/11] Use Mutex when modifying failedOperations, add exponential cooldown Just to be safe, won't allow concurrent goroutines to modify the map. The exponential cooldown prevents newer torrents with no seeds blocking older ones with seeds, when there are enough failures that a cooldown event would fill the queue with only failed torrents. --- config/metainfoFetcher.go | 22 +++++---- .../metainfoFetcher/metainfoFetcher.go | 47 ++++++++++++++++--- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/config/metainfoFetcher.go b/config/metainfoFetcher.go index 8afe58a5..a65f2859 100644 --- a/config/metainfoFetcher.go +++ b/config/metainfoFetcher.go @@ -1,22 +1,24 @@ package config type MetainfoFetcherConfig struct { - QueueSize int `json:"queue_size"` - Timeout int `json:"timeout"` - MaxDays int `json:"max_days"` - FailCooldown int `json:"fail_cooldown"` - WakeUpInterval int `json:"wake_up_interval"` + QueueSize int `json:"queue_size"` + Timeout int `json:"timeout"` + MaxDays int `json:"max_days"` + BaseFailCooldown int `json:"base_fail_cooldown"` + MaxFailCooldown int `json:"max_fail_cooldown"` + WakeUpInterval int `json:"wake_up_interval"` UploadRateLimiter int `json:"upload_rate_limiter"` DownloadRateLimiter int `json:"download_rate_limiter"` } var DefaultMetainfoFetcherConfig = MetainfoFetcherConfig{ - QueueSize: 10, - Timeout: 120, // 2 min - MaxDays: 90, - FailCooldown: 30 * 60, // in seconds, when failed torrents will be able to be fetched again. - WakeUpInterval: 300, // 5 min + QueueSize: 10, + Timeout: 120, // 2 min + MaxDays: 90, + BaseFailCooldown: 30 * 60, // in seconds, when failed torrents will be able to be fetched again. + MaxFailCooldown: 48 * 60 * 60, + WakeUpInterval: 300, // 5 min UploadRateLimiter: 1024, // kbps DownloadRateLimiter: 1024, diff --git a/service/torrent/metainfoFetcher/metainfoFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go index 179eca5f..60f37e2a 100644 --- a/service/torrent/metainfoFetcher/metainfoFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -10,6 +10,7 @@ import ( serviceBase "github.com/ewhal/nyaa/service" torrentService "github.com/ewhal/nyaa/service/torrent" "golang.org/x/time/rate" + "math" "sync" "time" ) @@ -20,11 +21,14 @@ type MetainfoFetcher struct { queueSize int timeout int maxDays int - failCooldown int + baseFailCooldown int + maxFailCooldown int done chan int queue []*FetchOperation queueMutex sync.Mutex failedOperations map[uint]time.Time + numFails map[uint]int + failsMutex sync.Mutex wakeUp *time.Ticker wg sync.WaitGroup } @@ -48,9 +52,11 @@ func New(fetcherConfig *config.MetainfoFetcherConfig) (fetcher *MetainfoFetcher, queueSize: fetcherConfig.QueueSize, timeout: fetcherConfig.Timeout, maxDays: fetcherConfig.MaxDays, - failCooldown: fetcherConfig.FailCooldown, + baseFailCooldown: fetcherConfig.BaseFailCooldown, + maxFailCooldown: fetcherConfig.MaxFailCooldown, done: make(chan int, 1), failedOperations: make(map[uint]time.Time), + numFails: make(map[uint]int), wakeUp: time.NewTicker(time.Second * time.Duration(fetcherConfig.WakeUpInterval)), } @@ -95,6 +101,26 @@ func (fetcher *MetainfoFetcher) removeFromQueue(op *FetchOperation) bool { return false } +func (fetcher *MetainfoFetcher) markAsFailed(tID uint) { + fetcher.failsMutex.Lock() + defer fetcher.failsMutex.Unlock() + + if n, ok := fetcher.numFails[tID]; ok { + fetcher.numFails[tID] = n + 1 + } else { + fetcher.numFails[tID] = 1 + } + + fetcher.failedOperations[tID] = time.Now() +} + +func (fetcher *MetainfoFetcher) removeFromFailed(tID uint) { + fetcher.failsMutex.Lock() + defer fetcher.failsMutex.Unlock() + + delete(fetcher.failedOperations, tID) +} + func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error { torrentFiles := info.UpvertedFiles() log.Infof("TID %d has %d files.", dbEntry.ID, len(torrentFiles)) @@ -152,7 +178,7 @@ func (fetcher *MetainfoFetcher) gotResult(r Result) { } if !updatedSuccessfully { - fetcher.failedOperations[r.operation.torrent.ID] = time.Now() + fetcher.markAsFailed(r.operation.torrent.ID) } fetcher.removeFromQueue(r.operation) @@ -160,16 +186,23 @@ func (fetcher *MetainfoFetcher) gotResult(r Result) { func (fetcher *MetainfoFetcher) removeOldFailures() { // Cooldown is disabled - if fetcher.failCooldown < 0 { + if fetcher.baseFailCooldown < 0 { return } + maxCd := time.Duration(fetcher.maxFailCooldown) * time.Second now := time.Now() for id, failTime := range fetcher.failedOperations { - if failTime.Add(time.Duration(fetcher.failCooldown) * time.Second).Before(now) { - log.Infof("Torrent TID %d gone through cooldown, removing from failures") + cdMult := int(math.Pow(2, float64(fetcher.numFails[id] - 1))) + cd := time.Duration(cdMult * fetcher.baseFailCooldown) * time.Second + if cd > maxCd { + cd = maxCd + } + + if failTime.Add(cd).Before(now) { + log.Infof("Torrent TID %d gone through cooldown, removing from failures", id) // Deleting keys inside a loop seems to be safe. - delete(fetcher.failedOperations, id) + fetcher.removeFromFailed(id) } } } From b9904161f05685856a19377d79a02b8f860f5887 Mon Sep 17 00:00:00 2001 From: ElegantMonkey Date: Sun, 14 May 2017 21:21:04 -0300 Subject: [PATCH 11/11] Only show added message if actually added to queue --- service/torrent/metainfoFetcher/metainfoFetcher.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/service/torrent/metainfoFetcher/metainfoFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go index 60f37e2a..8da54513 100644 --- a/service/torrent/metainfoFetcher/metainfoFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -247,10 +247,9 @@ func (fetcher *MetainfoFetcher) fillQueue() { continue } - log.Infof("Added TID %d for filesize fetching", T.ID) operation := NewFetchOperation(fetcher, T) - if fetcher.addToQueue(operation) { + log.Infof("Added TID %d for filesize fetching", T.ID) fetcher.wg.Add(1) go operation.Start(fetcher.results) } else {