diff --git a/config/default_config.yml b/config/default_config.yml index abad4a71..ba313ac1 100644 --- a/config/default_config.yml +++ b/config/default_config.yml @@ -124,6 +124,8 @@ torrents: torrents: # GenerationClientPort : Port used by the torrent client created during torrent generation generation_client_port: 50006 +# FilesFetchingClientPort: Port used by the client created by file fetching + files_fetching_client_port: 50005 # FileStorage : Location of folder that will contain generated torrent files filestorage: ./downloads/ # TorrentStorageLink : Url of torrent file download location (eg https://your.site/somewhere/%s) diff --git a/config/structs.go b/config/structs.go index a41016e4..b7065436 100644 --- a/config/structs.go +++ b/config/structs.go @@ -133,20 +133,21 @@ type TrackersConfig struct { // TorrentsConfig : Config struct for Torrents type TorrentsConfig struct { - Status []bool `yaml:"status,omitempty,omitempty"` - SukebeiCategories map[string]string `yaml:"sukebei_categories,omitempty"` - CleanCategories map[string]string `yaml:"clean_categories,omitempty"` - EnglishOnlyCategories ArrayString `yaml:"english_only_categories,omitempty"` - NonEnglishOnlyCategories ArrayString `yaml:"non_english_only_categories,omitempty"` - AdditionalLanguages ArrayString `yaml:"additional_languages,omitempty"` - FileStorage string `yaml:"filestorage,omitempty"` - StorageLink string `yaml:"storage_link,omitempty"` - CacheLink string `yaml:"cache_link,omitempty"` - Trackers TrackersConfig `yaml:"trackers,flow,omitempty"` - Order string `yaml:"order,omitempty"` - Sort string `yaml:"sort,omitempty"` - Tags Tags `yaml:"tags,flow,omitempty"` - GenerationClientPort int `yaml:"generation_client_port,flow,omitempty"` + Status []bool `yaml:"status,omitempty,omitempty"` + SukebeiCategories map[string]string `yaml:"sukebei_categories,omitempty"` + CleanCategories map[string]string `yaml:"clean_categories,omitempty"` + EnglishOnlyCategories ArrayString `yaml:"english_only_categories,omitempty"` + NonEnglishOnlyCategories ArrayString `yaml:"non_english_only_categories,omitempty"` + AdditionalLanguages ArrayString `yaml:"additional_languages,omitempty"` + FileStorage string `yaml:"filestorage,omitempty"` + StorageLink string `yaml:"storage_link,omitempty"` + CacheLink string `yaml:"cache_link,omitempty"` + Trackers TrackersConfig `yaml:"trackers,flow,omitempty"` + Order string `yaml:"order,omitempty"` + Sort string `yaml:"sort,omitempty"` + Tags Tags `yaml:"tags,flow,omitempty"` + GenerationClientPort int `yaml:"generation_client_port,flow,omitempty"` + FilesFetchingClientPort int `yaml:"files_fetching_client_port,flow,omitempty"` } // UploadConfig : Config struct for uploading torrents diff --git a/controllers/torrent/files.go b/controllers/torrent/files.go new file mode 100644 index 00000000..3aee1e21 --- /dev/null +++ b/controllers/torrent/files.go @@ -0,0 +1,78 @@ +package torrentController + +import ( + "html/template" + "encoding/hex" + "net/http" + "strings" + "strconv" + + "github.com/NyaaPantsu/nyaa/models/torrents" + "github.com/NyaaPantsu/nyaa/models" + "github.com/NyaaPantsu/nyaa/templates" + "github.com/NyaaPantsu/nyaa/utils/format" + "github.com/NyaaPantsu/nyaa/utils/filelist" + "github.com/Stephen304/goscrape" + "github.com/gin-gonic/gin" +) + +func GetFilesHandler(c *gin.Context) { + id, _ := strconv.ParseInt(c.Param("id"), 10, 32) + torrent, err := torrents.FindByID(uint(id)) + + if err != nil { + c.Status(http.StatusNotFound) + return + } + + + if len(torrent.FileList) == 0 { + var blankScrape models.Scrape + ScrapeFiles(format.InfoHashToMagnet(strings.TrimSpace(torrent.Hash), torrent.Name, GetTorrentTrackers(torrent)...), torrent, blankScrape, true) + } + + folder := filelist.FileListToFolder(torrent.FileList, "root") + templates.TorrentFileList(c, torrent.ToJSON(), folder) +} + +// ScrapeFiles : Scrape torrent files +func ScrapeFiles(magnet string, torrent *models.Torrent, currentStats models.Scrape, statsExists bool) (error, []FileJSON) { + if client == nil { + err := initClient() + if err != nil { + return err, []FileJSON{} + } + } + + t, _ := client.AddMagnet(magnet) + <-t.GotInfo() + + infoHash := t.InfoHash() + dst := make([]byte, hex.EncodedLen(len(t.InfoHash()))) + hex.Encode(dst, infoHash[:]) + + var UDP []string + + for _, tracker := range t.Metainfo().AnnounceList[0] { + if strings.HasPrefix(tracker, "udp") { + UDP = append(UDP, tracker) + } + } + var results goscrape.Result + if len(UDP) != 0 { + udpscrape := goscrape.NewBulk(UDP) + results = udpscrape.ScrapeBulk([]string{torrent.Hash})[0] + } + t.Drop() + return nil, UpdateTorrentStats(torrent, results, currentStats, t.Files(), statsExists) +} + +// FileJSON for file model in json, +type FileJSON struct { + Path string `json:"path"` + Filesize template.HTML `json:"filesize"` +} + +func fileSize(filesize int64) template.HTML { + return template.HTML(format.FileSize(filesize)) +} diff --git a/controllers/torrent/router.go b/controllers/torrent/router.go index ce5c067b..e315cd1b 100644 --- a/controllers/torrent/router.go +++ b/controllers/torrent/router.go @@ -8,6 +8,7 @@ import ( func init() { router.Get().Any("/download/:hash", DownloadTorrent) router.Get().Any("/stats/:id", GetStatsHandler) + router.Get().Any("/files/:id", GetFilesHandler) torrentRoutes := router.Get().Group("/torrent", middlewares.LoggedInMiddleware()) { diff --git a/controllers/torrent/stats.go b/controllers/torrent/stats.go index f11dffa6..0286dc45 100644 --- a/controllers/torrent/stats.go +++ b/controllers/torrent/stats.go @@ -1,6 +1,7 @@ package torrentController import ( + "path/filepath" "strconv" "strings" "net/url" @@ -9,10 +10,34 @@ import ( "github.com/NyaaPantsu/nyaa/models/torrents" "github.com/NyaaPantsu/nyaa/models" "github.com/NyaaPantsu/nyaa/config" + "github.com/NyaaPantsu/nyaa/utils/log" + "github.com/NyaaPantsu/nyaa/utils/format" "github.com/Stephen304/goscrape" "github.com/gin-gonic/gin" + + "github.com/anacrolix/dht" + "github.com/anacrolix/torrent" + "github.com/bradfitz/slice" ) +var client *torrent.Client + +func initClient() error { + clientConfig := torrent.Config{ + DHTConfig: dht.ServerConfig{ + StartingNodes: dht.GlobalBootstrapAddrs, + }, + ListenAddr: ":" + strconv.Itoa(config.Get().Torrents.FilesFetchingClientPort), + } + cl, err := torrent.NewClient(&clientConfig) + if err != nil { + log.Errorf("error creating client: %s", err) + return err + } + client = cl + return nil +} + // ViewHeadHandler : Controller for getting torrent stats func GetStatsHandler(c *gin.Context) { id, err := strconv.ParseInt(c.Param("id"), 10, 32) @@ -20,8 +45,8 @@ func GetStatsHandler(c *gin.Context) { return } - torrent, err := torrents.FindRawByID(uint(id)) - + updateTorrent, err := torrents.FindByID(uint(id)) + if err != nil { return } @@ -29,42 +54,42 @@ func GetStatsHandler(c *gin.Context) { var CurrentData models.Scrape statsExists := !(models.ORM.Where("torrent_id = ?", id).Find(&CurrentData).RecordNotFound()) - if statsExists { + if statsExists && c.Request.URL.Query()["files"] == nil { //Stats already exist, we check if the torrent stats have been scraped already very recently and if so, we stop there to avoid abuse of the /stats/:id route - if (CurrentData.Seeders == 0 && CurrentData.Leechers == 0 && CurrentData.Completed == 0) && time.Since(CurrentData.LastScrape).Minutes() <= config.Get().Scrape.MaxStatScrapingFrequencyUnknown { + if isEmptyScrape(CurrentData) && time.Since(CurrentData.LastScrape).Minutes() <= config.Get().Scrape.MaxStatScrapingFrequencyUnknown { //Unknown stats but has been scraped less than X minutes ago (X being the limit set in the config file) return } - if (CurrentData.Seeders != 0 || CurrentData.Leechers != 0 || CurrentData.Completed != 0) && time.Since(CurrentData.LastScrape).Minutes() <= config.Get().Scrape.MaxStatScrapingFrequency { + if !isEmptyScrape(CurrentData) && time.Since(CurrentData.LastScrape).Minutes() <= config.Get().Scrape.MaxStatScrapingFrequency { //Known stats but has been scraped less than X minutes ago (X being the limit set in the config file) return } } - var Trackers []string - if len(torrent.Trackers) > 3 { - for _, line := range strings.Split(torrent.Trackers[3:], "&tr=") { - tracker, error := url.QueryUnescape(line) - if error == nil && strings.HasPrefix(tracker, "udp") { - Trackers = append(Trackers, tracker) - } - //Cannot scrape from http trackers so don't put them in the array + Trackers := GetTorrentTrackers(updateTorrent) + + var stats goscrape.Result + var torrentFiles []FileJSON + + if c.Request.URL.Query()["files"] != nil { + if len(updateTorrent.FileList) > 0 { + return } + err, torrentFiles = ScrapeFiles(format.InfoHashToMagnet(strings.TrimSpace(updateTorrent.Hash), updateTorrent.Name, Trackers...), updateTorrent, CurrentData, statsExists) + if err != nil { + return + } + } else { + //Single() returns an array which contain results for each torrent Hash it is fed, since we only feed him one we want to directly access the results + stats = goscrape.Single(Trackers, []string{ + updateTorrent.Hash, + })[0] + UpdateTorrentStats(updateTorrent, stats, CurrentData, []torrent.File{}, statsExists) } - for _, tracker := range config.Get().Torrents.Trackers.Default { - if !contains(Trackers, tracker) && strings.HasPrefix(tracker, "udp") { - Trackers = append(Trackers, tracker) - } - } - - stats := goscrape.Single(Trackers, []string{ - torrent.Hash, - })[0] - //Single() returns an array which contain results for each torrent Hash it is fed, since we only feed him one we want to directly access the results //If we put seeders on -1, the script instantly knows the fetching did not give any result, avoiding having to check all three stats below and in view.jet.html's javascript - if stats.Seeders == 0 && stats.Leechers == 0 && stats.Completed == 0 { + if isEmptyResult(stats) { stats.Seeders = -1 } @@ -72,30 +97,89 @@ func GetStatsHandler(c *gin.Context) { "seeders": stats.Seeders, "leechers": stats.Leechers, "downloads": stats.Completed, + "filelist": torrentFiles, + "totalsize": fileSize(updateTorrent.Filesize), }) + return +} + +// UpdateTorrentStats : Update stats & filelist if files are specified, otherwise just stats +func UpdateTorrentStats(torrent *models.Torrent, stats goscrape.Result, currentStats models.Scrape, Files []torrent.File, statsExists bool) (JSONFilelist []FileJSON) { if stats.Seeders == -1 { stats.Seeders = 0 } if !statsExists { - torrent.Scrape = torrent.Scrape.Create(uint(id), uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now()) - //Create entry in the DB because none exist + torrent.Scrape = torrent.Scrape.Create(torrent.ID, uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now()) + //Create a stat entry in the DB because none exist } else { //Entry in the DB already exists, simply update it - if (CurrentData.Seeders == 0 && CurrentData.Leechers == 0 && CurrentData.Completed == 0) || (stats.Seeders != 0 && stats.Leechers != 0 && stats.Completed != 0 ) { - torrent.Scrape = &models.Scrape{uint(id), uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now()} + if isEmptyScrape(currentStats) || !isEmptyResult(stats) { + torrent.Scrape = &models.Scrape{torrent.ID, uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now()} } else { - torrent.Scrape = &models.Scrape{uint(id), uint32(CurrentData.Seeders), uint32(CurrentData.Leechers), uint32(CurrentData.Completed), time.Now()} + torrent.Scrape = &models.Scrape{torrent.ID, uint32(currentStats.Seeders), uint32(currentStats.Leechers), uint32(currentStats.Completed), time.Now()} } - //Only overwrite stats if the old one are Unknown OR if the current ones are not unknown, preventing good stats from being turned into unknown own but allowing good stats to be updated to more reliable ones + //Only overwrite stats if the old one are Unknown OR if the new ones are not unknown, preventing good stats from being turned into unknown but allowing good stats to be updated to more reliable ones torrent.Scrape.Update(false) + } + + if len(Files) > 1 { + files, err := torrent.CreateFileList(Files) + if err != nil { + return + } + + JSONFilelist = make([]FileJSON, 0, len(files)) + for _, f := range files { + JSONFilelist = append(JSONFilelist, FileJSON{ + Path: filepath.Join(f.Path()...), + Filesize: fileSize(f.Filesize), + }) + } + + // Sort file list by lowercase filename + slice.Sort(JSONFilelist, func(i, j int) bool { + return strings.ToLower(JSONFilelist[i].Path) < strings.ToLower(JSONFilelist[j].Path) + }) + } else if len(Files) == 1 { + torrent.Filesize = Files[0].Length() + torrent.Update(false) } return } +// GetTorrentTrackers : Get the torrent trackers and add the default ones if they are missing +func GetTorrentTrackers(torrent *models.Torrent) []string { + var Trackers []string + if len(torrent.Trackers) > 3 { + for _, line := range strings.Split(torrent.Trackers[3:], "&tr=") { + tracker, error := url.QueryUnescape(line) + if error == nil && strings.HasPrefix(tracker, "udp") { + Trackers = append(Trackers, tracker) + } + //Cannot scrape from http trackers only keep UDP ones + } + } + + for _, tracker := range config.Get().Torrents.Trackers.Default { + if !contains(Trackers, tracker) && strings.HasPrefix(tracker, "udp") { + Trackers = append(Trackers, tracker) + } + } + return Trackers +} + +func isEmptyResult(stats goscrape.Result) bool { + return stats.Seeders == 0 && stats.Leechers == 0 && stats.Completed == 0 +} + +func isEmptyScrape(stats models.Scrape) bool { + return stats.Seeders == 0 && stats.Leechers == 0 && stats.Completed == 0 +} + func contains(s []string, e string) bool { for _, a := range s { if a == e { diff --git a/models/file.go b/models/file.go index f1bd4d8c..31ee85c6 100644 --- a/models/file.go +++ b/models/file.go @@ -52,12 +52,18 @@ func (f *File) SetPath(path []string) error { // Filename : Returns the filename of the file func (f *File) Filename() string { path := f.Path() + if len(path) == 0 { + return "" + } return path[len(path)-1] } // FilenameWithoutExtension : Returns the filename of the file without the extension func (f *File) FilenameWithoutExtension() string { path := f.Path() + if len(path) == 0 { + return "" + } fileName := path[len(path)-1] index := strings.LastIndex(fileName, ".") @@ -71,10 +77,13 @@ func (f *File) FilenameWithoutExtension() string { // FilenameExtension : Returns the extension of a filename, or an empty string func (f *File) FilenameExtension() string { path := f.Path() + if len(path) == 0 { + return "" + } fileName := path[len(path)-1] index := strings.LastIndex(fileName, ".") - if index == -1 { + if index == -1 || index+1 == len(fileName){ return "" } diff --git a/models/torrent.go b/models/torrent.go index 890f10c2..09004d7c 100644 --- a/models/torrent.go +++ b/models/torrent.go @@ -22,6 +22,7 @@ import ( "github.com/NyaaPantsu/nyaa/utils/format" "github.com/NyaaPantsu/nyaa/utils/log" "github.com/NyaaPantsu/nyaa/utils/sanitize" + "github.com/anacrolix/torrent" "github.com/bradfitz/slice" "github.com/fatih/structs" ) @@ -460,6 +461,26 @@ func (t *Torrent) Update(unscope bool) (int, error) { return http.StatusOK, nil } +func (t *Torrent) CreateFileList(Files []torrent.File) ([]File, error) { + var createdFilelist []File + t.Filesize = 0 + + for _, uploadedFile := range Files { + file := File{TorrentID: t.ID, Filesize: uploadedFile.Length()} + err := file.SetPath(uploadedFile.FileInfo().Path) + if err != nil { + return []File{}, err + } + createdFilelist = append(createdFilelist, file) + t.Filesize += uploadedFile.Length() + ORM.Create(&file) + } + + t.FileList = createdFilelist + t.Update(false) + return createdFilelist, nil +} + // UpdateUnscope : Update a torrent based on model func (t *Torrent) UpdateUnscope() (int, error) { return t.Update(true) diff --git a/models/torrents/find.go b/models/torrents/find.go index 7f091ed5..d56af241 100644 --- a/models/torrents/find.go +++ b/models/torrents/find.go @@ -39,10 +39,8 @@ func FindByID(id uint) (*models.Torrent, error) { } - tmp := models.ORM.Where("torrent_id = ?", id).Preload("Scrape").Preload("Uploader").Preload("Comments") - if id > config.Get().Models.LastOldTorrentID { - tmp = tmp.Preload("FileList") - } + tmp := models.ORM.Where("torrent_id = ?", id).Preload("Scrape").Preload("Uploader").Preload("Comments").Preload("FileList") + if id <= config.Get().Models.LastOldTorrentID && !config.IsSukebei() { // only preload old comments if they could actually exist tmp = tmp.Preload("OldComments") diff --git a/templates/site/torrents/filelist.jet.html b/templates/site/torrents/filelist.jet.html new file mode 100644 index 00000000..53de4eb9 --- /dev/null +++ b/templates/site/torrents/filelist.jet.html @@ -0,0 +1,33 @@ +{{ extends "layouts/index_site" }} +{{ import "layouts/partials/helpers/csrf" }} +{{ import "layouts/partials/helpers/captcha" }} +{{ import "layouts/partials/helpers/errors" }} +{{ import "layouts/partials/helpers/tags" }} +{{ import "layouts/partials/helpers/treeview" }} +{{ import "layouts/partials/helpers/tag_form" }} +{{block title()}}{{Torrent.Name}}{{end}} +{{block content_body()}} +
+

{{T("torrent_filelist")}}

+ «- {{T("back_to_torrent", Torrent.Name)}}
+ + +
+ + + + + + + + + {{ if len(Torrent.FileList) > 0 }} + {{ yield make_treeview(treeviewData=makeTreeViewData(RootFolder, 0, "root")) }} + {{else}} + + {{end}} + +
{{ T("file_name")}}{{ T("size")}}
{{ T("no_files") }}
+
+
+{{end}} diff --git a/templates/site/torrents/view.jet.html b/templates/site/torrents/view.jet.html index ed2d6f84..713aff4f 100644 --- a/templates/site/torrents/view.jet.html +++ b/templates/site/torrents/view.jet.html @@ -52,7 +52,7 @@ {{ T("size")}}: - {{ fileSize(Torrent.Filesize, T, true) }} + {{ fileSize(Torrent.Filesize, T, true) }} {{ if len(Torrent.Languages) > 0 && Torrent.Languages[0] != "" }} @@ -205,10 +205,14 @@

{{ T("no_description") }}

{{end}} 0}}checked{{end}}/> - +
- {{ if len(Torrent.FileList) > 0 }} - {* how do i concat lol *} @@ -217,12 +221,13 @@ - {{ yield make_treeview(treeviewData=makeTreeViewData(RootFolder, 0, "root")) }} + {{ if len(Torrent.FileList) > 0 }} + {{ yield make_treeview(treeviewData=makeTreeViewData(RootFolder, 0, "root")) }} + {{else}} + + {{end}}
{{ T("no_files") }}
- {{ else }} -

{{ T("no_files") }}

- {{ end }}

{{ T("comments")}}

@@ -353,11 +358,40 @@ Modal.Init({ // order of apparition of the modals button: ["#reportPopup", "#tagPopup"] }); +{{ if len(Torrent.FileList) == 0 }} + var FileListContainer = document.querySelector("#filelist tbody"), + FileListLabel = document.getElementsByClassName("filelist-control")[0], + FileListOldHtml = FileListContainer.innerHTML + + FileListLabel.innerHTML = FileListLabel.innerText + + FileListLabel.addEventListener("click", function (e) { + FileListContainer.innerHTML = "{{T("loading_file_list")}}" + Query.Get('/stats/{{Torrent.ID}}?files', function (data) { + + if(data.totalsize != null && data.totalsize != "0.0 B") document.getElementsByClassName("torrent-info-size")[0].innerHTML = data.totalsize + if(data.filelist != null) { + FileListContainer.innerHTML = "" + FileListLabel.style.opacity = 1 + document.getElementById("filelist").style.opacity = 1 + + for(var i = 0; i < data.filelist.length; i++) { + var file = data.filelist[i] + if(file.filesize == "0.0 B") file.filesize = "{{T("unknown")}}" + FileListContainer.innerHTML = FileListContainer.innerHTML + ''+ file.path +''+ file.filesize +'' + } + } else { + FileListContainer.innerHTML = FileListOldHtml + } + }) + }) + {{end}} + -{{ if !torrentFileExists(Torrent.Hash, Torrent.TorrentLink)}} {{end}} {{if Torrent.StatsObsolete[1] }} - {{end}} + {{ if User.ID > 0 }} diff --git a/templates/template.go b/templates/template.go index 71ddc906..d036bd9b 100644 --- a/templates/template.go +++ b/templates/template.go @@ -160,6 +160,14 @@ func Torrent(c *gin.Context, torrent models.TorrentJSON, rootFolder *filelist.Fi Render(c, path.Join(SiteDir, "torrents", "view.jet.html"), variables) } +// Torrent render a torrent view template +func TorrentFileList(c *gin.Context, torrent models.TorrentJSON, rootFolder *filelist.FileListFolder) { + variables := Commonvariables(c) + variables.Set("Torrent", torrent) + variables.Set("RootFolder", rootFolder) + Render(c, path.Join(SiteDir, "torrents", "filelist.jet.html"), variables) +} + // userProfilBase render the base for user profile func userProfileBase(c *gin.Context, templateName string, userProfile *models.User, variables jet.VarMap) { currentUser, _, _ := cookies.CurrentUser(c) diff --git a/templates/template_test.go b/templates/template_test.go index 70a1b24e..830be0e7 100644 --- a/templates/template_test.go +++ b/templates/template_test.go @@ -110,6 +110,11 @@ func walkDirTest(dir string, t *testing.T) { variables.Set("RootFolder", filelist.FileListToFolder(fakeTorrent.FileList, "root")) return variables }, + "filelist.jet.html": func(variables jet.VarMap) jet.VarMap { + variables.Set("Torrent", fakeTorrent.ToJSON()) + variables.Set("RootFolder", filelist.FileListToFolder(fakeTorrent.FileList, "root")) + return variables + }, "settings.jet.html": func(variables jet.VarMap) jet.VarMap { variables.Set("Form", &LanguagesJSONResponse{"test", publicSettings.Languages{*fakeLanguage, *fakeLanguage}}) return variables diff --git a/translations/CHANGELOG.md b/translations/CHANGELOG.md index 1df53872..d351af40 100644 --- a/translations/CHANGELOG.md +++ b/translations/CHANGELOG.md @@ -101,6 +101,10 @@ ## 2017/11/04 * + nsfw_content * + generating_torrent_failed +## 2017/11/08 +* + loading_file_list +* + torrent_filelist +* + back_to_torrent ## 2017/11/09 * + userstatus_janitor * + ban diff --git a/translations/en-us.all.json b/translations/en-us.all.json index 7307e445..19440689 100644 --- a/translations/en-us.all.json +++ b/translations/en-us.all.json @@ -767,6 +767,18 @@ "id": "no_files", "translation": "No files found? That doesn't even make sense!" }, + { + "id": "loading_file_list", + "translation": "Loading the file list, long file lists can take time to fetch..." + }, + { + "id": "torrent_filelist", + "translation": "Torrent filelist" + }, + { + "id": "back_to_torrent", + "translation": "Back to \"%s\"" + }, { "id": "uploaded_by", "translation": "Uploaded by"