From 515f4696cc032b8dfffea07e1f2ed912f5822ab0 Mon Sep 17 00:00:00 2001 From: Eliot Whalan Date: Mon, 15 May 2017 19:32:28 +1000 Subject: [PATCH] Patch scrapers to support sukebei --- service/scraper/scraper.go | 2 +- service/scraper/transaction.go | 5 ++-- .../metainfoFetcher/metainfoFetcher.go | 29 +++++++++---------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/service/scraper/scraper.go b/service/scraper/scraper.go index 28797dd8..482c9540 100644 --- a/service/scraper/scraper.go +++ b/service/scraper/scraper.go @@ -181,7 +181,7 @@ func (sc *Scraper) Scrape(packets uint) { now := time.Now().Add(0 - sc.interval) // only scrape torretns uploaded within 90 days oldest := now.Add(0 - (time.Hour * 24 * 90)) - rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM torrents WHERE ( last_scrape IS NULL OR last_scrape < ? ) AND date > ? ORDER BY torrent_id DESC LIMIT ?", now, oldest, packets*ScrapesPerPacket).Rows() + rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM "+config.TableName+" WHERE ( last_scrape IS NULL OR last_scrape < ? ) AND date > ? ORDER BY torrent_id DESC LIMIT ?", now, oldest, packets*ScrapesPerPacket).Rows() if err == nil { counter := 0 var scrape [ScrapesPerPacket]model.Torrent diff --git a/service/scraper/transaction.go b/service/scraper/transaction.go index aaf6ece3..fb698769 100644 --- a/service/scraper/transaction.go +++ b/service/scraper/transaction.go @@ -6,6 +6,7 @@ import ( "net" "time" + "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/model" "github.com/ewhal/nyaa/util/log" @@ -53,8 +54,8 @@ func (t *Transaction) handleScrapeReply(data []byte) { } } -const pgQuery = "UPDATE torrents SET seeders = $1 , leechers = $2 , completed = $3 , last_scrape = $4 WHERE torrent_id = $5" -const sqliteQuery = "UPDATE torrents SET seeders = ? , leechers = ? , completed = ? , last_scrape = ? WHERE torrent_id = ?" +const pgQuery = "UPDATE " + config.TableName + " SET seeders = $1 , leechers = $2 , completed = $3 , last_scrape = $4 WHERE torrent_id = $5" +const sqliteQuery = "UPDATE " + config.TableName + " SET seeders = ? , leechers = ? , completed = ? , last_scrape = ? WHERE torrent_id = ?" // Sync syncs models with database func (t *Transaction) Sync() (err error) { diff --git a/service/torrent/metainfoFetcher/metainfoFetcher.go b/service/torrent/metainfoFetcher/metainfoFetcher.go index 8da54513..271a2e0f 100644 --- a/service/torrent/metainfoFetcher/metainfoFetcher.go +++ b/service/torrent/metainfoFetcher/metainfoFetcher.go @@ -1,4 +1,4 @@ -package metainfoFetcher; +package metainfoFetcher import ( "github.com/anacrolix/torrent" @@ -6,9 +6,9 @@ import ( "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/model" - "github.com/ewhal/nyaa/util/log" serviceBase "github.com/ewhal/nyaa/service" torrentService "github.com/ewhal/nyaa/service/torrent" + "github.com/ewhal/nyaa/util/log" "golang.org/x/time/rate" "math" "sync" @@ -38,10 +38,10 @@ func New(fetcherConfig *config.MetainfoFetcherConfig) (fetcher *MetainfoFetcher, // Well, it seems this is the right way to convert speed -> rate.Limiter // https://github.com/anacrolix/torrent/blob/master/cmd/torrent/main.go if fetcherConfig.UploadRateLimiter != -1 { - clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.UploadRateLimiter * 1024), 256<<10) + clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.UploadRateLimiter*1024), 256<<10) } if fetcherConfig.DownloadRateLimiter != -1 { - clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.DownloadRateLimiter * 1024), 1<<20) + clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.DownloadRateLimiter*1024), 1<<20) } client, err := torrent.NewClient(&clientConfig) @@ -78,7 +78,7 @@ func (fetcher *MetainfoFetcher) addToQueue(op *FetchOperation) bool { fetcher.queueMutex.Lock() defer fetcher.queueMutex.Unlock() - if len(fetcher.queue) + 1 > fetcher.queueSize { + if len(fetcher.queue)+1 > fetcher.queueSize { return false } @@ -86,7 +86,6 @@ func (fetcher *MetainfoFetcher) addToQueue(op *FetchOperation) bool { return true } - func (fetcher *MetainfoFetcher) removeFromQueue(op *FetchOperation) bool { fetcher.queueMutex.Lock() defer fetcher.queueMutex.Unlock() @@ -132,8 +131,8 @@ func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error { log.Infof("Adding file %s to filelist of TID %d", path, dbEntry.ID) dbFile := model.File{ TorrentID: dbEntry.ID, - Path: path, - Filesize: file.Length, + Path: path, + Filesize: file.Length, } err := db.ORM.Create(&dbFile).Error @@ -193,8 +192,8 @@ func (fetcher *MetainfoFetcher) removeOldFailures() { maxCd := time.Duration(fetcher.maxFailCooldown) * time.Second now := time.Now() for id, failTime := range fetcher.failedOperations { - cdMult := int(math.Pow(2, float64(fetcher.numFails[id] - 1))) - cd := time.Duration(cdMult * fetcher.baseFailCooldown) * time.Second + cdMult := int(math.Pow(2, float64(fetcher.numFails[id]-1))) + cd := time.Duration(cdMult*fetcher.baseFailCooldown) * time.Second if cd > maxCd { cd = maxCd } @@ -214,7 +213,7 @@ func (fetcher *MetainfoFetcher) fillQueue() { return } - oldest := time.Now().Add(0 - (time.Hour * time.Duration(24 * fetcher.maxDays))) + oldest := time.Now().Add(0 - (time.Hour * time.Duration(24*fetcher.maxDays))) excludedIDS := make([]uint, 0, len(fetcher.failedOperations)) for id, _ := range fetcher.failedOperations { excludedIDS = append(excludedIDS, id) @@ -223,11 +222,11 @@ func (fetcher *MetainfoFetcher) fillQueue() { // Nice query lol // Select the torrents with no filesize, or without any rows with torrent_id in the files table, that are younger than fetcher.MaxDays var params serviceBase.WhereParams - + if len(excludedIDS) > 0 { - params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ? AND torrent_id NOT IN (?)", oldest, excludedIDS) + params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR ("+config.TableName+".torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = "+config.TableName+".torrent_id))) AND date > ? AND torrent_id NOT IN (?)", oldest, excludedIDS) } else { - params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ?", oldest) + params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR ("+config.TableName+".torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = "+config.TableName+".torrent_id))) AND date > ?", oldest) } dbTorrents, err := torrentService.GetTorrentsOrderByNoCount(¶ms, "", fetcher.queueSize, 0) @@ -241,7 +240,6 @@ func (fetcher *MetainfoFetcher) fillQueue() { return } - for _, T := range dbTorrents { if fetcher.isFetchingOrFailed(T) { continue @@ -305,4 +303,3 @@ func (fetcher *MetainfoFetcher) Close() error { func (fetcher *MetainfoFetcher) Wait() { fetcher.wg.Wait() } -