Moving deadtrackers list to default config
Support torrent file creation when upload without scraping * Fix possible bug where no one is seeding a torrent yet (first time upload) Add primary tracker in torrent file
Cette révision appartient à :
akuma06
Parent
f5a1cef461
révision
19f39acb56
|
|
@ -145,6 +145,21 @@ torrents:
|
|||
needed:
|
||||
- 0
|
||||
- 12
|
||||
dead:
|
||||
- ://open.nyaatorrents.info:6544
|
||||
- ://tracker.openbittorrent.com:80
|
||||
- ://tracker.publicbt.com:80
|
||||
- ://stats.anisource.net:2710
|
||||
- ://exodus.desync.com
|
||||
- ://open.demonii.com:1337
|
||||
- ://tracker.istole.it:80
|
||||
- ://tracker.ccc.de:80
|
||||
- ://bt2.careland.com.cn:6969
|
||||
- ://announce.torrentsmd.com:8080
|
||||
- ://open.demonii.com:1337
|
||||
- ://tracker.btcake.com
|
||||
- ://tracker.prq.to
|
||||
- ://bt.rghost.net
|
||||
# TorrentOrder : Default sorting field for torrents
|
||||
order: date
|
||||
# TorrentSort : Default sorting order for torrents
|
||||
|
|
|
|||
|
|
@ -107,55 +107,56 @@ type I18nConfig struct {
|
|||
|
||||
// ScrapeConfig : Config struct for Scraping
|
||||
type ScrapeConfig struct {
|
||||
URL string `json:"scrape_url" yaml:"url,omitempty"`
|
||||
Name string `json:"name" yaml:"name,omitempty"`
|
||||
IntervalSeconds int64 `json:"interval" yaml:"interval,omitempty"`
|
||||
URL string `json:"scrape_url" yaml:"url,omitempty"`
|
||||
Name string `json:"name" yaml:"name,omitempty"`
|
||||
IntervalSeconds int64 `json:"interval" yaml:"interval,omitempty"`
|
||||
}
|
||||
|
||||
// ScraperConfig : Config struct for Scraper
|
||||
type ScraperConfig struct {
|
||||
Addr string `json:"bind" yaml:"addr,omitempty"`
|
||||
NumWorkers int `json:"workers" yaml:"workers,omitempty"`
|
||||
IntervalSeconds int64 `json:"default_interval" yaml:"default_interval,omitempty"`
|
||||
Trackers []ScrapeConfig `json:"trackers" yaml:"trackers,omitempty"`
|
||||
StatScrapingFrequency float64 `json:"stat_scraping_frequency" yaml:"stat_scraping_frequency,omitempty"`
|
||||
StatScrapingFrequencyUnknown float64 `json:"stat_scraping_frequency_unknown" yaml:"stat_scraping_frequency_unknown,omitempty"`
|
||||
MaxStatScrapingFrequency float64 `json:"max_stat_scraping_frequency" yaml:"max_stat_scraping_frequency,omitempty"`
|
||||
MaxStatScrapingFrequencyUnknown float64 `json:"max_stat_scraping_frequency_unknown" yaml:"max_stat_scraping_frequency_unknown,omitempty"`
|
||||
Addr string `json:"bind" yaml:"addr,omitempty"`
|
||||
NumWorkers int `json:"workers" yaml:"workers,omitempty"`
|
||||
IntervalSeconds int64 `json:"default_interval" yaml:"default_interval,omitempty"`
|
||||
Trackers []ScrapeConfig `json:"trackers" yaml:"trackers,omitempty"`
|
||||
StatScrapingFrequency float64 `json:"stat_scraping_frequency" yaml:"stat_scraping_frequency,omitempty"`
|
||||
StatScrapingFrequencyUnknown float64 `json:"stat_scraping_frequency_unknown" yaml:"stat_scraping_frequency_unknown,omitempty"`
|
||||
MaxStatScrapingFrequency float64 `json:"max_stat_scraping_frequency" yaml:"max_stat_scraping_frequency,omitempty"`
|
||||
MaxStatScrapingFrequencyUnknown float64 `json:"max_stat_scraping_frequency_unknown" yaml:"max_stat_scraping_frequency_unknown,omitempty"`
|
||||
}
|
||||
|
||||
// TrackersConfig ; Config struct for Trackers
|
||||
type TrackersConfig struct {
|
||||
Default ArrayString `yaml:"default,flow,omitempty"`
|
||||
NeededTrackers []int `yaml:"needed,flow,omitempty"`
|
||||
DeadTrackers ArrayString `yaml:"dead,flow,omitempty"`
|
||||
}
|
||||
|
||||
// TorrentsConfig : Config struct for Torrents
|
||||
type TorrentsConfig struct {
|
||||
Status []bool `yaml:"status,omitempty,omitempty"`
|
||||
SukebeiCategories map[string]string `yaml:"sukebei_categories,omitempty"`
|
||||
CleanCategories map[string]string `yaml:"clean_categories,omitempty"`
|
||||
EnglishOnlyCategories ArrayString `yaml:"english_only_categories,omitempty"`
|
||||
NonEnglishOnlyCategories ArrayString `yaml:"non_english_only_categories,omitempty"`
|
||||
AdditionalLanguages ArrayString `yaml:"additional_languages,omitempty"`
|
||||
FileStorage string `yaml:"filestorage,omitempty"`
|
||||
StorageLink string `yaml:"storage_link,omitempty"`
|
||||
CacheLink string `yaml:"cache_link,omitempty"`
|
||||
Trackers TrackersConfig `yaml:"trackers,flow,omitempty"`
|
||||
Order string `yaml:"order,omitempty"`
|
||||
Sort string `yaml:"sort,omitempty"`
|
||||
Tags Tags `yaml:"tags,flow,omitempty"`
|
||||
GenerationClientPort int `yaml:"generation_client_port,flow,omitempty"`
|
||||
Status []bool `yaml:"status,omitempty,omitempty"`
|
||||
SukebeiCategories map[string]string `yaml:"sukebei_categories,omitempty"`
|
||||
CleanCategories map[string]string `yaml:"clean_categories,omitempty"`
|
||||
EnglishOnlyCategories ArrayString `yaml:"english_only_categories,omitempty"`
|
||||
NonEnglishOnlyCategories ArrayString `yaml:"non_english_only_categories,omitempty"`
|
||||
AdditionalLanguages ArrayString `yaml:"additional_languages,omitempty"`
|
||||
FileStorage string `yaml:"filestorage,omitempty"`
|
||||
StorageLink string `yaml:"storage_link,omitempty"`
|
||||
CacheLink string `yaml:"cache_link,omitempty"`
|
||||
Trackers TrackersConfig `yaml:"trackers,flow,omitempty"`
|
||||
Order string `yaml:"order,omitempty"`
|
||||
Sort string `yaml:"sort,omitempty"`
|
||||
Tags Tags `yaml:"tags,flow,omitempty"`
|
||||
GenerationClientPort int `yaml:"generation_client_port,flow,omitempty"`
|
||||
}
|
||||
|
||||
// UploadConfig : Config struct for uploading torrents
|
||||
type UploadConfig struct {
|
||||
DefaultAnidexToken string `yaml:"anidex_api_token,omitempty"`
|
||||
DefaultNyaasiToken string `yaml:"nyaasi_api_token,omitempty"`
|
||||
DefaultTokyoTToken string `yaml:"tokyot_api_token,omitempty"`
|
||||
UploadsDisabled bool `yaml:"uploads_disabled,omitempty"`
|
||||
AdminsAreStillAllowedTo bool `yaml:"admins_are_still_allowed_to,omitempty"`
|
||||
TrustedUsersAreStillAllowedTo bool `yaml:"trusted_users_are_still_allowed_to,omitempty"`
|
||||
DefaultAnidexToken string `yaml:"anidex_api_token,omitempty"`
|
||||
DefaultNyaasiToken string `yaml:"nyaasi_api_token,omitempty"`
|
||||
DefaultTokyoTToken string `yaml:"tokyot_api_token,omitempty"`
|
||||
UploadsDisabled bool `yaml:"uploads_disabled,omitempty"`
|
||||
AdminsAreStillAllowedTo bool `yaml:"admins_are_still_allowed_to,omitempty"`
|
||||
TrustedUsersAreStillAllowedTo bool `yaml:"trusted_users_are_still_allowed_to,omitempty"`
|
||||
}
|
||||
|
||||
// UsersConfig : Config struct for Users
|
||||
|
|
@ -224,9 +225,9 @@ type ModelsConfig struct {
|
|||
}
|
||||
|
||||
type DefaultThemeConfig struct {
|
||||
Theme string `yaml:"theme,omitempty"`
|
||||
Dark string `yaml:"dark,omitempty"`
|
||||
Forced string `yaml:"forced,omitempty"`
|
||||
Theme string `yaml:"theme,omitempty"`
|
||||
Dark string `yaml:"dark,omitempty"`
|
||||
Forced string `yaml:"forced,omitempty"`
|
||||
}
|
||||
|
||||
// SearchConfig : Config struct for search
|
||||
|
|
@ -274,3 +275,14 @@ func (ty TagTypes) Get(tagType string) TagType {
|
|||
}
|
||||
return TagType{}
|
||||
}
|
||||
|
||||
// GetDefault returns the first tracker from the needed ones
|
||||
func (tc TrackersConfig) GetDefault() string {
|
||||
if len(tc.NeededTrackers) > 0 {
|
||||
return tc.Default[tc.NeededTrackers[0]]
|
||||
}
|
||||
if len(tc.Default) > 0 {
|
||||
return tc.Default[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ func ExtractInfo(c *gin.Context, r *torrentValidator.TorrentRequest) error {
|
|||
return err
|
||||
}
|
||||
|
||||
_, err = r.ValidateMultipartUpload(c, uploadFormTorrent)
|
||||
err = r.ValidateMultipartUpload(c, uploadFormTorrent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -129,7 +129,7 @@ func ExtractInfo(c *gin.Context, r *torrentValidator.TorrentRequest) error {
|
|||
return err
|
||||
}
|
||||
// We are not saving the file here because we need to add our own tracker list to the torrent file, therefore, we generate the torrent file at upload time through GenerateTorrent()
|
||||
// when it is magnet or torrent file upload
|
||||
// when it is magnet
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,3 +15,4 @@ var errTorrentHashInvalid = errors.New("torrent_hash_invalid")
|
|||
var errTorrentTagsInvalid = errors.New("torrent_tags_invalid")
|
||||
var errMissingFieldConfig = errors.New("Torrent Configuration in config.yml is invalid, missing a 'field' attribute in tags types")
|
||||
var errWrongFieldConfig = errors.New("Torrent Configuration in config.yml is invalid, wrong 'field' attribute in tags types")
|
||||
var errInvalidTorrentFile = errors.New("torrent_file_invalid")
|
||||
|
|
|
|||
|
|
@ -4,9 +4,8 @@ import (
|
|||
"encoding/base32"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
|
|
@ -15,13 +14,11 @@ import (
|
|||
"github.com/NyaaPantsu/nyaa/config"
|
||||
"github.com/NyaaPantsu/nyaa/utils/categories"
|
||||
"github.com/NyaaPantsu/nyaa/utils/cookies"
|
||||
"github.com/NyaaPantsu/nyaa/utils/format"
|
||||
msg "github.com/NyaaPantsu/nyaa/utils/messages"
|
||||
"github.com/NyaaPantsu/nyaa/utils/metainfo"
|
||||
"github.com/NyaaPantsu/nyaa/utils/torrentLanguages"
|
||||
"github.com/NyaaPantsu/nyaa/utils/validator/tag"
|
||||
"github.com/anacrolix/torrent/metainfo"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/zeebo/bencode"
|
||||
)
|
||||
|
||||
// ValidateName is a function validating the torrent name
|
||||
|
|
@ -191,80 +188,90 @@ func (r *TorrentRequest) ExtractLanguage() error {
|
|||
}
|
||||
|
||||
// ValidateMultipartUpload : Check if multipart upload is valid
|
||||
func (r *TorrentRequest) ValidateMultipartUpload(c *gin.Context, uploadFormTorrent string) (multipart.File, error) {
|
||||
func (r *TorrentRequest) ValidateMultipartUpload(c *gin.Context, uploadFormTorrent string) error {
|
||||
// first: parse torrent file (if any) to fill missing information
|
||||
tfile, _, err := c.Request.FormFile(uploadFormTorrent)
|
||||
if err == nil {
|
||||
var torrent metainfo.TorrentFile
|
||||
|
||||
// decode torrent
|
||||
_, seekErr := tfile.Seek(0, io.SeekStart)
|
||||
if seekErr != nil {
|
||||
return tfile, seekErr
|
||||
}
|
||||
err = bencode.NewDecoder(tfile).Decode(&torrent)
|
||||
torrent, err := metainfo.Load(tfile)
|
||||
if err != nil {
|
||||
return tfile, metainfo.ErrInvalidTorrentFile
|
||||
return err
|
||||
}
|
||||
|
||||
torrentInfos, err := torrent.UnmarshalInfo()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check a few things
|
||||
if torrent.IsPrivate() {
|
||||
return tfile, errTorrentPrivate
|
||||
if *torrentInfos.Private {
|
||||
return errTorrentPrivate
|
||||
}
|
||||
trackers := torrent.GetAllAnnounceURLS()
|
||||
r.Trackers = CheckTrackers(trackers)
|
||||
// We check that the trackers are valid,
|
||||
// we filter the dead ones
|
||||
// and we add the needed ones from our config in the request and the torrent file
|
||||
r.Trackers = CheckTrackers(torrent)
|
||||
if len(r.Trackers) == 0 {
|
||||
return tfile, errTorrentNoTrackers
|
||||
return errTorrentNoTrackers
|
||||
}
|
||||
|
||||
// Name
|
||||
// For the name, if none is provided in request, we set the name from the torrent file
|
||||
if len(r.Name) == 0 {
|
||||
r.Name = torrent.TorrentName()
|
||||
r.Name = torrentInfos.Name
|
||||
}
|
||||
|
||||
// Magnet link: if a file is provided it should be empty
|
||||
if len(r.Magnet) != 0 {
|
||||
return tfile, errTorrentAndMagnet
|
||||
return errTorrentAndMagnet
|
||||
}
|
||||
|
||||
_, seekErr = tfile.Seek(0, io.SeekStart)
|
||||
if seekErr != nil {
|
||||
return tfile, seekErr
|
||||
// We are using here default functions from anacrolix
|
||||
infohash := torrent.HashInfoBytes()
|
||||
if infohash.String() == "" {
|
||||
return errInvalidTorrentFile
|
||||
}
|
||||
infohash, err := metainfo.DecodeInfohash(tfile)
|
||||
if err != nil {
|
||||
return tfile, metainfo.ErrInvalidTorrentFile
|
||||
}
|
||||
r.Infohash = infohash
|
||||
r.Magnet = format.InfoHashToMagnet(infohash, r.Name, trackers...)
|
||||
r.Infohash = infohash.String()
|
||||
r.Magnet = torrent.Magnet(r.Name, infohash).String()
|
||||
|
||||
// extract filesize
|
||||
r.Filesize = int64(torrent.TotalSize())
|
||||
r.Filesize = torrentInfos.TotalLength()
|
||||
|
||||
// extract filelist
|
||||
fileInfos := torrent.Info.GetFiles()
|
||||
fileInfos := torrentInfos.Files
|
||||
for _, fileInfo := range fileInfos {
|
||||
r.FileList = append(r.FileList, uploadedFile{
|
||||
Path: fileInfo.Path,
|
||||
Filesize: int64(fileInfo.Length),
|
||||
Filesize: fileInfo.Length,
|
||||
})
|
||||
}
|
||||
// Since we can change with anacrolix the trackers of a torrent,
|
||||
// We will use this to generate directly the torrent file here
|
||||
file := fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, infohash.String())
|
||||
f, err := os.Create(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
err = torrent.Write(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
} else {
|
||||
err = r.ValidateMagnet()
|
||||
if err != nil {
|
||||
return tfile, err
|
||||
return err
|
||||
}
|
||||
err = r.ValidateHash()
|
||||
if err != nil {
|
||||
return tfile, err
|
||||
return err
|
||||
}
|
||||
// TODO: Get Trackers from magnet URL
|
||||
r.Filesize = 0
|
||||
r.Filepath = ""
|
||||
|
||||
return tfile, nil
|
||||
return nil
|
||||
}
|
||||
return tfile, err
|
||||
return err
|
||||
}
|
||||
|
||||
// ExtractInfo : Function to assign values from request to ReassignForm
|
||||
|
|
|
|||
|
|
@ -3,42 +3,50 @@ package torrentValidator
|
|||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/NyaaPantsu/nyaa/config"
|
||||
"github.com/anacrolix/torrent/metainfo"
|
||||
)
|
||||
|
||||
// CheckTrackers : Check if there is good trackers in torrent
|
||||
func CheckTrackers(trackers []string) []string {
|
||||
func CheckTrackers(t *metainfo.MetaInfo) []string {
|
||||
// TODO: move to runtime configuration
|
||||
var deadTrackers = []string{ // substring matches!
|
||||
"://open.nyaatorrents.info:6544",
|
||||
"://tracker.openbittorrent.com:80",
|
||||
"://tracker.publicbt.com:80",
|
||||
"://stats.anisource.net:2710",
|
||||
"://exodus.desync.com",
|
||||
"://open.demonii.com:1337",
|
||||
"://tracker.istole.it:80",
|
||||
"://tracker.ccc.de:80",
|
||||
"://bt2.careland.com.cn:6969",
|
||||
"://announce.torrentsmd.com:8080",
|
||||
"://open.demonii.com:1337",
|
||||
"://tracker.btcake.com",
|
||||
"://tracker.prq.to",
|
||||
"://bt.rghost.net"}
|
||||
var deadTrackers = config.Get().Torrents.Trackers.DeadTrackers
|
||||
|
||||
var trackerRet []string
|
||||
for _, t := range trackers {
|
||||
urlTracker, err := url.Parse(t)
|
||||
if err == nil {
|
||||
good := true
|
||||
for _, check := range deadTrackers {
|
||||
if strings.Contains(t, check) {
|
||||
good = false
|
||||
break // No need to continue the for loop
|
||||
tempList := t.AnnounceList
|
||||
for kgroup, group := range tempList {
|
||||
for ktracker, tracker := range group {
|
||||
urlTracker, err := url.Parse(tracker)
|
||||
if err == nil {
|
||||
good := true
|
||||
for _, check := range deadTrackers {
|
||||
// the tracker is part of the deadtracker list
|
||||
// we don't keep it
|
||||
if strings.Contains(tracker, check) {
|
||||
good = false
|
||||
break // No need to continue the for loop
|
||||
}
|
||||
}
|
||||
if good {
|
||||
// We only keep the good trackers
|
||||
trackerRet = append(trackerRet, urlTracker.String())
|
||||
} else {
|
||||
// if the tracker is no good, we remove it from the group
|
||||
t.AnnounceList[kgroup] = append(t.AnnounceList[kgroup][:ktracker], t.AnnounceList[kgroup][ktracker+1:]...)
|
||||
}
|
||||
}
|
||||
if good {
|
||||
trackerRet = append(trackerRet, urlTracker.String())
|
||||
}
|
||||
}
|
||||
|
||||
// We need to update the group of the trackers
|
||||
// if there is no good trackers in this group, we remove the group
|
||||
if len(t.AnnounceList[kgroup]) == 0 {
|
||||
t.AnnounceList = append(t.AnnounceList[:kgroup], t.AnnounceList[kgroup+1:]...)
|
||||
}
|
||||
}
|
||||
defaultTracker := config.Get().Torrents.Trackers.GetDefault()
|
||||
if defaultTracker != "" {
|
||||
t.Announce = defaultTracker
|
||||
}
|
||||
return trackerRet
|
||||
}
|
||||
|
|
|
|||
Référencer dans un nouveau ticket