Albirew/nyaa-pantsu
Albirew/nyaa-pantsu
Archivé
1
0
Bifurcation 0
Code Activité

Patch scrapers to support sukebei

Parcourir la source
Cette révision appartient à :
Eliot Whalan 2017-05-15 19:32:28 +10:00
Parent b93ec8f7b9
révision 515f4696cc
3 fichiers modifiés avec 17 ajouts et 19 suppressions
Voir les statistiques Télécharger le patch en tant que fichier Télécharger le fichier diff Développer tous les fichiers Réduire tous les fichiers

2
service/scraper/scraper.go
Voir le fichier

@ -181,7 +181,7 @@ func (sc *Scraper) Scrape(packets uint) {
now := time.Now().Add(0 - sc.interval)
// only scrape torretns uploaded within 90 days
oldest := now.Add(0 - (time.Hour * 24 * 90))
rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM torrents WHERE ( last_scrape IS NULL OR last_scrape < ? ) AND date > ? ORDER BY torrent_id DESC LIMIT ?", now, oldest, packets*ScrapesPerPacket).Rows()
rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM "+config.TableName+" WHERE ( last_scrape IS NULL OR last_scrape < ? ) AND date > ? ORDER BY torrent_id DESC LIMIT ?", now, oldest, packets*ScrapesPerPacket).Rows()
if err == nil {
counter := 0
var scrape [ScrapesPerPacket]model.Torrent

5
service/scraper/transaction.go
Voir le fichier

@ -6,6 +6,7 @@ import (
"net"
"time"
"github.com/ewhal/nyaa/config"
"github.com/ewhal/nyaa/db"
"github.com/ewhal/nyaa/model"
"github.com/ewhal/nyaa/util/log"
@ -53,8 +54,8 @@ func (t *Transaction) handleScrapeReply(data []byte) {
}
}
const pgQuery = "UPDATE torrents SET seeders = $1 , leechers = $2 , completed = $3 , last_scrape = $4 WHERE torrent_id = $5"
const sqliteQuery = "UPDATE torrents SET seeders = ? , leechers = ? , completed = ? , last_scrape = ? WHERE torrent_id = ?"
const pgQuery = "UPDATE " + config.TableName + " SET seeders = $1 , leechers = $2 , completed = $3 , last_scrape = $4 WHERE torrent_id = $5"
const sqliteQuery = "UPDATE " + config.TableName + " SET seeders = ? , leechers = ? , completed = ? , last_scrape = ? WHERE torrent_id = ?"
// Sync syncs models with database
func (t *Transaction) Sync() (err error) {

11
service/torrent/metainfoFetcher/metainfoFetcher.go
Voir le fichier

@ -1,4 +1,4 @@
package metainfoFetcher;
package metainfoFetcher
import (
"github.com/anacrolix/torrent"
@ -6,9 +6,9 @@ import (
"github.com/ewhal/nyaa/config"
"github.com/ewhal/nyaa/db"
"github.com/ewhal/nyaa/model"
"github.com/ewhal/nyaa/util/log"
serviceBase "github.com/ewhal/nyaa/service"
torrentService "github.com/ewhal/nyaa/service/torrent"
"github.com/ewhal/nyaa/util/log"
"golang.org/x/time/rate"
"math"
"sync"
@ -86,7 +86,6 @@ func (fetcher *MetainfoFetcher) addToQueue(op *FetchOperation) bool {
return true
}
func (fetcher *MetainfoFetcher) removeFromQueue(op *FetchOperation) bool {
fetcher.queueMutex.Lock()
defer fetcher.queueMutex.Unlock()
@ -225,9 +224,9 @@ func (fetcher *MetainfoFetcher) fillQueue() {
var params serviceBase.WhereParams
if len(excludedIDS) > 0 {
params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ? AND torrent_id NOT IN (?)", oldest, excludedIDS)
params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR ("+config.TableName+".torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = "+config.TableName+".torrent_id))) AND date > ? AND torrent_id NOT IN (?)", oldest, excludedIDS)
} else {
params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR (torrents.torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = torrents.torrent_id))) AND date > ?", oldest)
params = serviceBase.CreateWhereParams("((filesize IS NULL OR filesize = 0) OR ("+config.TableName+".torrent_id NOT IN (SELECT files.torrent_id FROM files WHERE files.torrent_id = "+config.TableName+".torrent_id))) AND date > ?", oldest)
}
dbTorrents, err := torrentService.GetTorrentsOrderByNoCount(&params, "", fetcher.queueSize, 0)
@ -241,7 +240,6 @@ func (fetcher *MetainfoFetcher) fillQueue() {
return
}
for _, T := range dbTorrents {
if fetcher.isFetchingOrFailed(T) {
continue
@ -305,4 +303,3 @@ func (fetcher *MetainfoFetcher) Close() error {
func (fetcher *MetainfoFetcher) Wait() {
fetcher.wg.Wait()
}