Add Upload and Download limiters, rely on unique_index for avoiding duplicates

The FileList that is read with GetTorrents is empty, as it's only loaded with
GetTorrentById. So, always insert the new FileList on the database, and rely on
the unique_index to avoid duplications.

config/metainfoFetcher.go

@@ -5,6 +5,9 @@ type MetainfoFetcherConfig struct {
 	Timeout        int `json:"timeout"`
 	MaxDays        int `json:"max_days"`
 	WakeUpInterval int `json:"wake_up_interval"`
+
+	UploadRateLimiter   int `json:"upload_rate_limiter"`
+	DownloadRateLimiter int `json:"download_rate_limiter"`
 }
 
 var DefaultMetainfoFetcherConfig = MetainfoFetcherConfig{
@@ -12,5 +15,8 @@ var DefaultMetainfoFetcherConfig = MetainfoFetcherConfig{
 	Timeout:        120, // 2 min
 	MaxDays:        90,
 	WakeUpInterval: 300, // 5 min
+
+	UploadRateLimiter:   1024,
+	DownloadRateLimiter: 1024,
 }
 

model/file.go

@@ -2,8 +2,8 @@ package model
 
 type File struct {
 	ID        uint   `gorm:"column:file_id;primary_key"`
-	TorrentID uint   `gorm:"column:torrent_id"`
+	TorrentID uint   `gorm:"column:torrent_id;unique_index:idx_tid_path"`
-	Path      string `gorm:"column:path"`
+	Path      string `gorm:"column:path;unique_index:idx_tid_path"`
 	Filesize  int64  `gorm:"column:filesize"`
 }
 

service/torrent/metainfoFetcher/metainfoFetcher.go

@@ -9,6 +9,7 @@ import (
 	"github.com/ewhal/nyaa/util/log"
 	serviceBase "github.com/ewhal/nyaa/service"
 	torrentService "github.com/ewhal/nyaa/service/torrent"
+	"golang.org/x/time/rate"
 	"sync"
 	"time"
 )
@@ -28,7 +29,18 @@ type MetainfoFetcher struct {
 }
 
 func New(fetcherConfig *config.MetainfoFetcherConfig) (fetcher *MetainfoFetcher, err error) {
-	client, err := torrent.NewClient(nil)
+	clientConfig := torrent.Config{}
+	// Well, it seems this is the right way to convert speed -> rate.Limiter
+	// https://github.com/anacrolix/torrent/blob/master/cmd/torrent/main.go
+	if fetcherConfig.UploadRateLimiter != -1 {
+		clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.UploadRateLimiter * 1024), 256<<10)
+	}
+	if fetcherConfig.DownloadRateLimiter != -1 {
+		clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.DownloadRateLimiter * 1024), 1<<20)
+	}
+
+	client, err := torrent.NewClient(&clientConfig)
+
 	fetcher = &MetainfoFetcher{
 		torrentClient:    client,
 		results:          make(chan Result, fetcherConfig.QueueSize),
@@ -86,15 +98,9 @@ func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error {
 	log.Infof("TID %d has %d files.", dbEntry.ID, len(torrentFiles))
 	for _, file := range torrentFiles {
 		path := file.DisplayPath(info)
-		fileExists := false
-		for _, existingFile := range dbEntry.FileList {
-			if existingFile.Path == path {
-				fileExists = true
-				break
-			}
-		}
 
-		if !fileExists {
+		// Can't read FileList from the GetTorrents output, rely on the unique_index
+		// to ensure no files are duplicated.
 		log.Infof("Adding file %s to filelist of TID %d", path, dbEntry.ID)
 		dbFile := model.File{
 			TorrentID: dbEntry.ID,
@@ -107,7 +113,6 @@ func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error {
 			return err
 		}
 	}
-	}
 
 	return nil
 }
@@ -223,6 +228,7 @@ func (fetcher *MetainfoFetcher) Close() error {
 	}
 
 	fetcher.done <- 1
+	log.Infof("Send done signal to everyone, waiting...")
 	fetcher.wg.Wait()
 	return nil
 }