Albirew/nyaa-pantsu
Archivé
1
0
Bifurcation 0

Add Upload and Download limiters, rely on unique_index for avoiding duplicates

The FileList that is read with GetTorrents is empty, as it's only loaded with
GetTorrentById. So, always insert the new FileList on the database, and rely on
the unique_index to avoid duplications.
Cette révision appartient à :
ElegantMonkey 2017-05-14 11:35:03 -03:00
Parent 839e6068df
révision 7d8cbe1393
3 fichiers modifiés avec 36 ajouts et 24 suppressions

Voir le fichier

@ -5,12 +5,18 @@ type MetainfoFetcherConfig struct {
Timeout int `json:"timeout"` Timeout int `json:"timeout"`
MaxDays int `json:"max_days"` MaxDays int `json:"max_days"`
WakeUpInterval int `json:"wake_up_interval"` WakeUpInterval int `json:"wake_up_interval"`
UploadRateLimiter int `json:"upload_rate_limiter"`
DownloadRateLimiter int `json:"download_rate_limiter"`
} }
var DefaultMetainfoFetcherConfig = MetainfoFetcherConfig{ var DefaultMetainfoFetcherConfig = MetainfoFetcherConfig{
QueueSize: 10, QueueSize: 10,
Timeout: 120, // 2 min Timeout: 120, // 2 min
MaxDays: 90, MaxDays: 90,
WakeUpInterval: 300, // 5 min WakeUpInterval: 300, // 5 min
UploadRateLimiter: 1024,
DownloadRateLimiter: 1024,
} }

Voir le fichier

@ -2,8 +2,8 @@ package model
type File struct { type File struct {
ID uint `gorm:"column:file_id;primary_key"` ID uint `gorm:"column:file_id;primary_key"`
TorrentID uint `gorm:"column:torrent_id"` TorrentID uint `gorm:"column:torrent_id;unique_index:idx_tid_path"`
Path string `gorm:"column:path"` Path string `gorm:"column:path;unique_index:idx_tid_path"`
Filesize int64 `gorm:"column:filesize"` Filesize int64 `gorm:"column:filesize"`
} }

Voir le fichier

@ -9,6 +9,7 @@ import (
"github.com/ewhal/nyaa/util/log" "github.com/ewhal/nyaa/util/log"
serviceBase "github.com/ewhal/nyaa/service" serviceBase "github.com/ewhal/nyaa/service"
torrentService "github.com/ewhal/nyaa/service/torrent" torrentService "github.com/ewhal/nyaa/service/torrent"
"golang.org/x/time/rate"
"sync" "sync"
"time" "time"
) )
@ -28,7 +29,18 @@ type MetainfoFetcher struct {
} }
func New(fetcherConfig *config.MetainfoFetcherConfig) (fetcher *MetainfoFetcher, err error) { func New(fetcherConfig *config.MetainfoFetcherConfig) (fetcher *MetainfoFetcher, err error) {
client, err := torrent.NewClient(nil) clientConfig := torrent.Config{}
// Well, it seems this is the right way to convert speed -> rate.Limiter
// https://github.com/anacrolix/torrent/blob/master/cmd/torrent/main.go
if fetcherConfig.UploadRateLimiter != -1 {
clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.UploadRateLimiter * 1024), 256<<10)
}
if fetcherConfig.DownloadRateLimiter != -1 {
clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.DownloadRateLimiter * 1024), 1<<20)
}
client, err := torrent.NewClient(&clientConfig)
fetcher = &MetainfoFetcher{ fetcher = &MetainfoFetcher{
torrentClient: client, torrentClient: client,
results: make(chan Result, fetcherConfig.QueueSize), results: make(chan Result, fetcherConfig.QueueSize),
@ -86,26 +98,19 @@ func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error {
log.Infof("TID %d has %d files.", dbEntry.ID, len(torrentFiles)) log.Infof("TID %d has %d files.", dbEntry.ID, len(torrentFiles))
for _, file := range torrentFiles { for _, file := range torrentFiles {
path := file.DisplayPath(info) path := file.DisplayPath(info)
fileExists := false
for _, existingFile := range dbEntry.FileList { // Can't read FileList from the GetTorrents output, rely on the unique_index
if existingFile.Path == path { // to ensure no files are duplicated.
fileExists = true log.Infof("Adding file %s to filelist of TID %d", path, dbEntry.ID)
break dbFile := model.File{
} TorrentID: dbEntry.ID,
Path: path,
Filesize: file.Length,
} }
if !fileExists { err := db.ORM.Create(&dbFile).Error
log.Infof("Adding file %s to filelist of TID %d", path, dbEntry.ID) if err != nil {
dbFile := model.File{ return err
TorrentID: dbEntry.ID,
Path: path,
Filesize: file.Length,
}
err := db.ORM.Create(&dbFile).Error
if err != nil {
return err
}
} }
} }
@ -223,6 +228,7 @@ func (fetcher *MetainfoFetcher) Close() error {
} }
fetcher.done <- 1 fetcher.done <- 1
log.Infof("Send done signal to everyone, waiting...")
fetcher.wg.Wait() fetcher.wg.Wait()
return nil return nil
} }