Albirew/nyaa-pantsu
Albirew/nyaa-pantsu
Archivé
1
0
Bifurcation 0
Code Activité

Patch scrapers to support sukebei

Parcourir la source
Cette révision appartient à :
Eliot Whalan 2017-05-15 19:32:28 +10:00
Parent b93ec8f7b9
révision 515f4696cc
3 fichiers modifiés avec 17 ajouts et 19 suppressions
Voir les statistiques Télécharger le patch en tant que fichier Télécharger le fichier diff Développer tous les fichiers Réduire tous les fichiers

2
service/scraper/scraper.go
Voir le fichier

@ -181,7 +181,7 @@ func (sc *Scraper) Scrape(packets uint) {
now := time.Now().Add(0 - sc.interval)
// only scrape torretns uploaded within 90 days
oldest := now.Add(0 - (time.Hour * 24 * 90))
rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM torrents WHERE ( last_scrape IS NULL OR last_scrape < ? ) AND date > ? ORDER BY torrent_id DESC LIMIT ?", now, oldest, packets*ScrapesPerPacket).Rows()
rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM "+config.TableName+" WHERE ( last_scrape IS NULL OR last_scrape < ? ) AND date > ? ORDER BY torrent_id DESC LIMIT ?", now, oldest, packets*ScrapesPerPacket).Rows()
if err == nil {
counter := 0
var scrape [ScrapesPerPacket]model.Torrent

5
service/scraper/transaction.go
Voir le fichier

@ -6,6 +6,7 @@ import (
"net"
"time"
"github.com/ewhal/nyaa/config"
"github.com/ewhal/nyaa/db"
"github.com/ewhal/nyaa/model"
"github.com/ewhal/nyaa/util/log"
@ -53,8 +54,8 @@ func (t *Transaction) handleScrapeReply(data []byte) {
}
}
const pgQuery = "UPDATE torrents SET seeders = $1 , leechers = $2 , completed = $3 , last_scrape = $4 WHERE torrent_id = $5"
const sqliteQuery = "UPDATE torrents SET seeders = ? , leechers = ? , completed = ? , last_scrape = ? WHERE torrent_id = ?"
const pgQuery = "UPDATE " + config.TableName + " SET seeders = $1 , leechers = $2 , completed = $3 , last_scrape = $4 WHERE torrent_id = $5"
const sqliteQuery = "UPDATE " + config.TableName + " SET seeders = ? , leechers = ? , completed = ? , last_scrape = ? WHERE torrent_id = ?"
// Sync syncs models with database
func (t *Transaction) Sync() (err error) {

29
service/torrent/metainfoFetcher/metainfoFetcher.go
Voir le fichier

@ -1,4 +1,4 @@
package metainfoFetcher;
package metainfoFetcher
import (
"github.com/anacrolix/torrent"
@ -6,9 +6,9 @@ import (
"github.com/ewhal/nyaa/config"
"github.com/ewhal/nyaa/db"
"github.com/ewhal/nyaa/model"
"github.com/ewhal/nyaa/util/log"
serviceBase "github.com/ewhal/nyaa/service"
torrentService "github.com/ewhal/nyaa/service/torrent"
"github.com/ewhal/nyaa/util/log"
"golang.org/x/time/rate"
"math"
"sync"
@ -38,10 +38,10 @@ func New(fetcherConfig *config.MetainfoFetcherConfig) (fetcher *MetainfoFetcher,
// Well, it seems this is the right way to convert speed -> rate.Limiter
// https://github.com/anacrolix/torrent/blob/master/cmd/torrent/main.go
if fetcherConfig.UploadRateLimiter != -1 {
clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.UploadRateLimiter * 1024), 256<<10)
clientConfig.UploadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.UploadRateLimiter*1024), 256<<10)
}
if fetcherConfig.DownloadRateLimiter != -1 {
clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.DownloadRateLimiter * 1024), 1<<20)
clientConfig.DownloadRateLimiter = rate.NewLimiter(rate.Limit(fetcherConfig.DownloadRateLimiter*1024), 1<<20)
}
client, err := torrent.NewClient(&clientConfig)
@ -78,7 +78,7 @@ func (fetcher *MetainfoFetcher) addToQueue(op *FetchOperation) bool {
fetcher.queueMutex.Lock()
defer fetcher.queueMutex.Unlock()
if len(fetcher.queue) + 1 > fetcher.queueSize {
if len(fetcher.queue)+1 > fetcher.queueSize {
return false
}
@ -86,7 +86,6 @@ func (fetcher *MetainfoFetcher) addToQueue(op *FetchOperation) bool {
return true
}
func (fetcher *MetainfoFetcher) removeFromQueue(op *FetchOperation) bool {
fetcher.queueMutex.Lock()
defer fetcher.queueMutex.Unlock()
@ -132,8 +131,8 @@ func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error {
log.Infof("Adding file %s to filelist of TID %d", path, dbEntry.ID)
dbFile := model.File{
TorrentID: dbEntry.ID,
Path: path,
Filesize: file.Length,
Path: path,
Filesize: file.Length,
}
err := db.ORM.Create(&dbFile).Error
@ -193,8 +192,8 @@ func (fetcher *MetainfoFetcher) removeOldFailures() {
maxCd := time.Duration(fetcher.maxFailCooldown) * time.Second
now := time.Now()
for id, failTime := range fetcher.failedOperations {
cdMult := int(math.Pow(2, float64(fetcher.numFails[id] - 1)))
cd := time.Duration(cdMult * fetcher.baseFailCooldown) * time.Second
cdMult := int(math.Pow(2, float64(fetcher.numFails[id]-1)))
cd := time.Duration(cdMult*fetcher.baseFailCooldown) * time.Second
if cd > maxCd {
cd = maxCd
}
@ -214,7 +213,7 @@ func (fetcher *MetainfoFetcher) fillQueue() {
return
}
oldest := time.Now().Add(0 - (time.Hour * time.Duration(24 * fetcher.maxDays)))
oldest := time.Now().Add(0 - (time.Hour * time.Duration(24*fetcher.maxDays)))
excludedIDS := make([]uint, 0, len(fetcher.failedOperations))
for id, _ := range fetcher.failedOperations {
excludedIDS = append(excludedIDS, id)
@ -223,11 +222,11 @@ func (fetcher *MetainfoFetcher) fillQueue() {
// Nice query lol
// Select the torrents with no filesize, or without any rows with torrent_id in the files table, that are younger than fetcher.MaxDays
var params serviceBase.WhereParams
if len(excludedIDS) > 0 {
params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ? AND torrent_id NOT IN (?)", oldest, excludedIDS)
params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR ("+config.TableName+".torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = "+config.TableName+".torrent_id))) AND date > ? AND torrent_id NOT IN (?)", oldest, excludedIDS)
} else {
params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ?", oldest)
params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR ("+config.TableName+".torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = "+config.TableName+".torrent_id))) AND date > ?", oldest)
}
dbTorrents, err := torrentService.GetTorrentsOrderByNoCount(&params, "", fetcher.queueSize, 0)
@ -241,7 +240,6 @@ func (fetcher *MetainfoFetcher) fillQueue() {
return
}
for _, T := range dbTorrents {
if fetcher.isFetchingOrFailed(T) {
continue
@ -305,4 +303,3 @@ func (fetcher *MetainfoFetcher) Close() error {
func (fetcher *MetainfoFetcher) Wait() {
fetcher.wg.Wait()
}