diff --git a/README.md b/README.md index e26d1bd5..5658b3bd 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ The aim of this project is to write a fully featured nyaa replacement in golang that anyone will be able to deploy locally or remotely. +## [Roadmap](https://trello.com/b/gMJBwoRq/nyaa-pantsu-cat-roadmap) +The Roadmap will give you an overview of the features and tasks that the project are currently discussing, working on and have completed. +If you are looking for a feature that is not listed just make a GitHub Issue and it will get added to the trello board. + +You can view the public trello board [here](https://trello.com/b/gMJBwoRq/nyaa-pantsu-cat-roadmap) or click on the "Roadmap". + # Requirements * Golang diff --git a/config/config.go b/config/config.go index 19b446d9..a47a4a93 100644 --- a/config/config.go +++ b/config/config.go @@ -21,7 +21,7 @@ type Config struct { DBType string `json:"db_type"` // DBParams will be directly passed to Gorm, and its internal // structure depends on the dialect for each db type - DBParams string `json:"db_params"` + DBParams string `json:"db_params"` DBLogMode string `json:"db_logmode"` // tracker scraper config (required) Scrape ScraperConfig `json:"scraper"` @@ -31,9 +31,14 @@ type Config struct { Search SearchConfig `json:"search"` // optional i2p configuration I2P *I2PConfig `json:"i2p"` + // filesize fetcher config + FilesizeFetcher FilesizeFetcherConfig `json:"filesize_fetcher"` + // internationalization config + I18n I18nConfig `json:"i18n"` } -var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", "default", DefaultScraperConfig, DefaultCacheConfig, DefaultSearchConfig, nil} +var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", "default", DefaultScraperConfig, DefaultCacheConfig, DefaultSearchConfig, nil, DefaultFilesizeFetcherConfig, DefaultI18nConfig} + var allowedDatabaseTypes = map[string]bool{ "sqlite3": true, @@ -57,6 +62,8 @@ func New() *Config { config.DBLogMode = Defaults.DBLogMode config.Scrape = Defaults.Scrape config.Cache = Defaults.Cache + config.FilesizeFetcher = Defaults.FilesizeFetcher + config.I18n = Defaults.I18n return &config } diff --git a/config/filesizeFetcher.go b/config/filesizeFetcher.go new file mode 100644 index 00000000..c86082fb --- /dev/null +++ b/config/filesizeFetcher.go @@ -0,0 +1,16 @@ +package config + +type FilesizeFetcherConfig struct { + QueueSize int `json:"queue_size"` + Timeout int `json:"timeout"` + MaxDays int `json:"max_days"` + WakeUpInterval int `json:"wake_up_interval"` +} + +var DefaultFilesizeFetcherConfig = FilesizeFetcherConfig{ + QueueSize: 10, + Timeout: 120, // 2 min + MaxDays: 90, + WakeUpInterval: 300, // 5 min +} + diff --git a/config/i18n.go b/config/i18n.go new file mode 100644 index 00000000..f0f67a3a --- /dev/null +++ b/config/i18n.go @@ -0,0 +1,11 @@ +package config + +type I18nConfig struct { + TranslationsDirectory string `json:"translations_directory"` + DefaultLanguage string `json:"default_language"` +} + +var DefaultI18nConfig = I18nConfig{ + TranslationsDirectory: "translations", + DefaultLanguage: "en-us", // TODO: Remove refs to "en-us" from the code and templates +} diff --git a/db/gorm.go b/db/gorm.go index c2e75e60..88c7db2c 100644 --- a/db/gorm.go +++ b/db/gorm.go @@ -58,7 +58,7 @@ func GormInit(conf *config.Config, logger Logger) (*gorm.DB, error) { if db.Error != nil { return db, db.Error } - db.AutoMigrate(&model.Torrent{}, &model.TorrentReport{}) + db.AutoMigrate(&model.Torrent{}, &model.TorrentReport{}, &model.File{}) if db.Error != nil { return db, db.Error } diff --git a/main.go b/main.go index 5b56615a..14f1cb0d 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ import ( "net/http" "os" - "path/filepath" "time" "github.com/ewhal/nyaa/cache" @@ -15,31 +14,21 @@ import ( "github.com/ewhal/nyaa/network" "github.com/ewhal/nyaa/router" "github.com/ewhal/nyaa/service/scraper" + "github.com/ewhal/nyaa/service/torrent/filesizeFetcher" + "github.com/ewhal/nyaa/util/languages" "github.com/ewhal/nyaa/util/log" "github.com/ewhal/nyaa/util/search" "github.com/ewhal/nyaa/util/signals" - "github.com/nicksnyder/go-i18n/i18n" ) -func initI18N() { - /* Initialize the languages translation */ - i18n.MustLoadTranslationFile("translations/en-us.all.json") - paths, err := filepath.Glob("translations/*.json") - if err == nil { - for _, path := range paths { - i18n.LoadTranslationFile(path) - } - } -} - // RunServer runs webapp mainloop func RunServer(conf *config.Config) { http.Handle("/", router.Router) // Set up server, srv := &http.Server{ - WriteTimeout: 24 * time.Second, - ReadTimeout: 8 * time.Second, + WriteTimeout: 5 * time.Second, + ReadTimeout: 5 * time.Second, } l, err := network.CreateHTTPListener(conf) log.CheckError(err) @@ -94,11 +83,24 @@ func RunScraper(conf *config.Config) { scraper.Wait() } +// RunFilesizeFetcher runs the database filesize fetcher main loop +func RunFilesizeFetcher(conf *config.Config) { + fetcher, err := filesizeFetcher.New(&conf.FilesizeFetcher) + if err != nil { + log.Fatalf("failed to start fetcher, %s", err) + return + } + + signals.RegisterCloser(fetcher) + fetcher.RunAsync() + fetcher.Wait() +} + func main() { conf := config.New() processFlags := conf.BindFlags() defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout") - mode := flag.String("mode", "webapp", "which mode to run daemon in, either webapp or scraper") + mode := flag.String("mode", "webapp", "which mode to run daemon in, either webapp, scraper or filesize_fetcher") flag.Float64Var(&conf.Cache.Size, "c", config.DefaultCacheSize, "size of the search cache in MB") flag.Parse() @@ -122,7 +124,10 @@ func main() { if err != nil { log.Fatal(err.Error()) } - initI18N() + err = languages.InitI18n(conf.I18n) + if err != nil { + log.Fatal(err.Error()) + } err = cache.Configure(&conf.Cache) if err != nil { log.Fatal(err.Error()) @@ -142,6 +147,8 @@ func main() { RunScraper(conf) } else if *mode == "webapp" { RunServer(conf) + } else if *mode == "metadata_fetcher" { + RunFilesizeFetcher(conf) } else { log.Fatalf("invalid runtime mode: %s", *mode) } diff --git a/model/file.go b/model/file.go new file mode 100644 index 00000000..4a95c6dc --- /dev/null +++ b/model/file.go @@ -0,0 +1,16 @@ +package model + +type File struct { + ID uint `gorm:"column:file_id;primary_key"` + TorrentID uint `gorm:"column:torrent_id"` + Path string `gorm:"column:path"` + Filesize int64 `gorm:"column:filesize"` + + Torrent *Torrent `gorm:"AssociationForeignKey:TorrentID;ForeignKey:torrent_id"` +} + +// Returns the total size of memory allocated for this struct +func (f File) Size() int { + return (1 + len(f.Path) + 2) * 8; +} + diff --git a/model/torrent.go b/model/torrent.go index 4db568d3..50e83031 100644 --- a/model/torrent.go +++ b/model/torrent.go @@ -44,6 +44,7 @@ type Torrent struct { Leechers uint32 `gorm:"column:leechers"` Completed uint32 `gorm:"column:completed"` LastScrape time.Time `gorm:"column:last_scrape"` + FileList []File `gorm:"ForeignKey:torrent_id"` } // Returns the total size of memory recursively allocated for this struct @@ -88,6 +89,11 @@ type CommentJSON struct { Date time.Time `json:"date"` } +type FileJSON struct { + Path string `json:"path"` + Length int64 `json:"length"` +} + type TorrentJSON struct { ID string `json:"id"` Name string `json:"name"` @@ -110,6 +116,7 @@ type TorrentJSON struct { Leechers uint32 `json:"leechers"` Completed uint32 `json:"completed"` LastScrape time.Time `json:"last_scrape"` + FileList []File `json:"file_list"` } // ToJSON converts a model.Torrent to its equivalent JSON structure @@ -155,6 +162,7 @@ func (t *Torrent) ToJSON() TorrentJSON { Seeders: t.Seeders, Completed: t.Completed, LastScrape: t.LastScrape, + FileList: t.FileList, } return res diff --git a/public/css/style.css b/public/css/style.css index e6c98d07..4c71284d 100644 --- a/public/css/style.css +++ b/public/css/style.css @@ -202,6 +202,10 @@ div.container div.blockBody:nth-of-type(2) table tr:first-of-type th:last-of-typ { width: 12rem; } +#mainmenu .navbar-form select.form-control#max +{ + width: 8rem; +} .special-img { position: relative; @@ -220,6 +224,7 @@ div.container div.blockBody:nth-of-type(2) table tr:first-of-type th:last-of-typ #mainmenu .badgemenu { padding-top: 0; + margin-right: -50px; /* don't ask */ } /* PROFILE PAGE */ diff --git a/public/js/main.js b/public/js/main.js index fbf5b3b9..d0e90e8a 100644 --- a/public/js/main.js +++ b/public/js/main.js @@ -2,9 +2,9 @@ var night = localStorage.getItem("night"); function toggleNightMode() { var night = localStorage.getItem("night"); if(night == "true") { - document.getElementById("style-dark").remove() + document.getElementsByTagName("head")[0].removeChild(darkStyleLink); } else { - document.getElementsByTagName("head")[0].append(darkStyleLink); + document.getElementsByTagName("head")[0].appendChild(darkStyleLink); } localStorage.setItem("night", (night == "true") ? "false" : "true"); } @@ -78,5 +78,4 @@ function loadLanguages() { xhr.send() } -loadLanguages(); - +loadLanguages(); \ No newline at end of file diff --git a/router/faqHandler.go b/router/faqHandler.go index af77167e..c1be82b3 100644 --- a/router/faqHandler.go +++ b/router/faqHandler.go @@ -8,11 +8,8 @@ import ( ) func FaqHandler(w http.ResponseWriter, r *http.Request) { - searchForm := NewSearchForm() - searchForm.HideAdvancedSearch = true - languages.SetTranslationFromRequest(faqTemplate, r, "en-us") - err := faqTemplate.ExecuteTemplate(w, "index.html", FaqTemplateVariables{Navigation{}, searchForm, GetUser(r), r.URL, mux.CurrentRoute(r)}) + err := faqTemplate.ExecuteTemplate(w, "index.html", FaqTemplateVariables{Navigation{}, NewSearchForm(), GetUser(r), r.URL, mux.CurrentRoute(r)}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/router/modpanel.go b/router/modpanel.go index 5446e00b..04c67528 100644 --- a/router/modpanel.go +++ b/router/modpanel.go @@ -5,6 +5,7 @@ import ( "html" "net/http" "strconv" + "strings" "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/model" @@ -21,6 +22,83 @@ import ( "github.com/gorilla/mux" ) +type ReassignForm struct { + AssignTo uint + By string + Data string + + Torrents []uint +} + +func (f *ReassignForm) ExtractInfo(r *http.Request) error { + f.By = r.FormValue("by") + if f.By != "olduser" && f.By != "torrentid" { + return fmt.Errorf("what?") + } + + f.Data = strings.Trim(r.FormValue("data"), " \r\n") + if f.By == "olduser" { + if f.Data == "" { + return fmt.Errorf("No username given") + } else if strings.Contains(f.Data, "\n") { + return fmt.Errorf("More than one username given") + } + } else if f.By == "torrentid" { + if f.Data == "" { + return fmt.Errorf("No IDs given") + } + splitData := strings.Split(f.Data, "\n") + for i, tmp := range splitData { + tmp = strings.Trim(tmp, " \r") + torrent_id, err := strconv.ParseUint(tmp, 10, 0) + if err != nil { + return fmt.Errorf("Couldn't parse number on line %d", i+1) + } + f.Torrents = append(f.Torrents, uint(torrent_id)) + } + } + + tmp := r.FormValue("to") + parsed, err := strconv.ParseUint(tmp, 10, 0) + if err != nil { + return err + } + f.AssignTo = uint(parsed) + _, _, _, _, err = userService.RetrieveUser(r, tmp) + if err != nil { + return fmt.Errorf("User to assign to doesn't exist") + } + + return nil +} + +func (f *ReassignForm) ExecuteAction() (int, error) { + + var toBeChanged []uint + var err error + if f.By == "olduser" { + toBeChanged, err = userService.RetrieveOldUploadsByUsername(f.Data) + if err != nil { + return 0, err + } + } else if f.By == "torrentid" { + toBeChanged = f.Torrents + } + + num := 0 + for _, torrent_id := range toBeChanged { + torrent, err2 := torrentService.GetRawTorrentById(torrent_id) + if err2 == nil { + torrent.UploaderID = f.AssignTo + db.ORM.Save(&torrent) + num += 1 + } + } + // TODO: clean shit from user_uploads_old if needed + return num, nil +} + + func IndexModPanel(w http.ResponseWriter, r *http.Request) { currentUser := GetUser(r) if userPermission.HasAdmin(currentUser) { @@ -32,7 +110,9 @@ func IndexModPanel(w http.ResponseWriter, r *http.Request) { torrentReports, _, _ := reportService.GetAllTorrentReports(offset, 0) languages.SetTranslationFromRequest(panelIndex, r, "en-us") - htv := PanelIndexVbs{torrents, model.TorrentReportsToJSON(torrentReports), users, comments, NewSearchForm(), currentUser, r.URL} + search := NewSearchForm() + search.ShowItemsPerPage = false + htv := PanelIndexVbs{torrents, model.TorrentReportsToJSON(torrentReports), users, comments, search, currentUser, r.URL} err := panelIndex.ExecuteTemplate(w, "admin_index.html", htv) log.CheckError(err) } else { @@ -59,9 +139,9 @@ func TorrentsListPanel(w http.ResponseWriter, r *http.Request) { searchParam, torrents, _, err := search.SearchByQuery(r, pagenum) searchForm := SearchForm{ - SearchParam: searchParam, - Category: searchParam.Category.String(), - HideAdvancedSearch: false, + SearchParam: searchParam, + Category: searchParam.Category.String(), + ShowItemsPerPage: true, } languages.SetTranslationFromRequest(panelTorrentList, r, "en-us") @@ -271,3 +351,43 @@ func TorrentReportDeleteModPanel(w http.ResponseWriter, r *http.Request) { http.Error(w, "admins only", http.StatusForbidden) } } + +func TorrentReassignModPanel(w http.ResponseWriter, r *http.Request) { + currentUser := GetUser(r) + if !userPermission.HasAdmin(currentUser) { + http.Error(w, "admins only", http.StatusForbidden) + return + } + languages.SetTranslationFromRequest(panelTorrentReassign, r, "en-us") + + htv := PanelTorrentReassignVbs{ReassignForm{}, NewSearchForm(), currentUser, form.NewErrors(), form.NewInfos(), r.URL} + err := panelTorrentReassign.ExecuteTemplate(w, "admin_index.html", htv) + log.CheckError(err) +} + +func TorrentPostReassignModPanel(w http.ResponseWriter, r *http.Request) { + currentUser := GetUser(r) + if !userPermission.HasAdmin(currentUser) { + http.Error(w, "admins only", http.StatusForbidden) + return + } + var rForm ReassignForm + err := form.NewErrors() + infos := form.NewInfos() + + err2 := rForm.ExtractInfo(r) + if err2 != nil { + err["errors"] = append(err["errors"], err2.Error()) + } else { + count, err2 := rForm.ExecuteAction() + if err2 != nil { + err["errors"] = append(err["errors"], "Something went wrong") + } else { + infos["infos"] = append(infos["infos"], fmt.Sprintf("%d torrents updated.", count)) + } + } + + htv := PanelTorrentReassignVbs{rForm, NewSearchForm(), currentUser, err, infos, r.URL} + err_ := panelTorrentReassign.ExecuteTemplate(w, "admin_index.html", htv) + log.CheckError(err_) +} diff --git a/router/notFoundHandler.go b/router/notFoundHandler.go index ed41e902..5480068f 100644 --- a/router/notFoundHandler.go +++ b/router/notFoundHandler.go @@ -10,11 +10,8 @@ import ( func NotFoundHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) - searchForm := NewSearchForm() - searchForm.HideAdvancedSearch = true - languages.SetTranslationFromRequest(notFoundTemplate, r, "en-us") - err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, searchForm, GetUser(r), r.URL, mux.CurrentRoute(r)}) + err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, NewSearchForm(), GetUser(r), r.URL, mux.CurrentRoute(r)}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/router/router.go b/router/router.go index 5964bac3..339eb7e3 100755 --- a/router/router.go +++ b/router/router.go @@ -101,17 +101,14 @@ func init() { Router.HandleFunc("/mod/torrent/delete", TorrentDeleteModPanel).Name("mod_tdelete") Router.HandleFunc("/mod/report/delete", TorrentReportDeleteModPanel).Name("mod_trdelete") Router.HandleFunc("/mod/comment/delete", CommentDeleteModPanel).Name("mod_cdelete") + Router.HandleFunc("/mod/reassign", TorrentReassignModPanel).Name("mod_treassign").Methods("GET") + Router.HandleFunc("/mod/reassign", TorrentPostReassignModPanel).Name("mod_treassign").Methods("POST") //reporting a torrent Router.HandleFunc("/report/{id}", ReportTorrentHandler).Methods("POST").Name("post_comment") Router.PathPrefix("/captcha").Methods("GET").HandlerFunc(captcha.ServeFiles) - //Router.HandleFunc("/report/create", gzipTorrentReportCreateHandler).Name("torrent_report_create").Methods("POST") - // TODO Allow only moderators to access /moderation/* - //Router.HandleFunc("/moderation/report/delete", gzipTorrentReportDeleteHandler).Name("torrent_report_delete").Methods("POST") - //Router.HandleFunc("/moderation/torrent/delete", gzipTorrentDeleteHandler).Name("torrent_delete").Methods("POST") - Router.HandleFunc("/language", SeeLanguagesHandler).Methods("GET").Name("see_languages") Router.HandleFunc("/language", ChangeLanguageHandler).Methods("POST").Name("change_language") diff --git a/router/searchHandler.go b/router/searchHandler.go index 57bfe1f1..1fe0e125 100644 --- a/router/searchHandler.go +++ b/router/searchHandler.go @@ -38,9 +38,9 @@ func SearchHandler(w http.ResponseWriter, r *http.Request) { navigationTorrents := Navigation{nbTorrents, int(searchParam.Max), pagenum, "search_page"} // Convert back to strings for now. searchForm := SearchForm{ - SearchParam: searchParam, - Category: searchParam.Category.String(), - HideAdvancedSearch: false, + SearchParam: searchParam, + Category: searchParam.Category.String(), + ShowItemsPerPage: true, } htv := HomeTemplateVariables{b, searchForm, navigationTorrents, GetUser(r), r.URL, mux.CurrentRoute(r)} diff --git a/router/template.go b/router/template.go index dab38d86..46b0a528 100644 --- a/router/template.go +++ b/router/template.go @@ -9,7 +9,7 @@ var TemplateDir = "templates" var homeTemplate, searchTemplate, faqTemplate, uploadTemplate, viewTemplate, viewRegisterTemplate, viewLoginTemplate, viewRegisterSuccessTemplate, viewVerifySuccessTemplate, viewProfileTemplate, viewProfileEditTemplate, viewUserDeleteTemplate, notFoundTemplate, changeLanguageTemplate *template.Template -var panelIndex, panelTorrentList, panelUserList, panelCommentList, panelTorrentEd, panelTorrentReportList *template.Template +var panelIndex, panelTorrentList, panelUserList, panelCommentList, panelTorrentEd, panelTorrentReportList, panelTorrentReassign *template.Template type templateLoader struct { templ **template.Template @@ -127,6 +127,11 @@ func ReloadTemplates() { name: "torrent_report", file: filepath.Join("admin", "torrent_report.html"), }, + templateLoader{ + templ: &panelTorrentReassign, + name: "torrent_reassign", + file: filepath.Join("admin", "reassign.html"), + }, } for idx := range modTempls { diff --git a/router/templateFunctions.go b/router/templateFunctions.go index d1b3fe54..5789d194 100644 --- a/router/templateFunctions.go +++ b/router/templateFunctions.go @@ -36,6 +36,24 @@ var FuncMap = template.FuncMap{ } return "error" }, + "genSearchWithOrdering": func(currentUrl url.URL, sortBy string) template.URL { + values := currentUrl.Query() + order := false + if _, ok := values["order"]; ok { + order, _ = strconv.ParseBool(values["order"][0]) + if values["sort"][0]==sortBy { + order=!order //Flip order by repeat-clicking + } else { + order=false //Default to descending when sorting by something new + } + } + values.Set("sort", sortBy) + values.Set("order", strconv.FormatBool(order)) + + currentUrl.RawQuery=values.Encode() + + return template.URL(currentUrl.String()) + }, "genNav": func(nav Navigation, currentUrl *url.URL, pagesSelectable int) template.HTML { var ret = "" if (nav.TotalItem > 0) { diff --git a/router/templateVariables.go b/router/templateVariables.go index 5e424088..d3217fe2 100644 --- a/router/templateVariables.go +++ b/router/templateVariables.go @@ -157,6 +157,7 @@ type PanelCommentListVbs struct { User *model.User URL *url.URL // For parsing Url in templates } + type PanelTorrentEdVbs struct { Upload UploadForm Search SearchForm @@ -174,6 +175,15 @@ type PanelTorrentReportListVbs struct { URL *url.URL // For parsing Url in templates } +type PanelTorrentReassignVbs struct { + Reassign ReassignForm + Search SearchForm // unused? + User *model.User // unused? + FormErrors map[string][]string + FormInfos map[string][]string + URL *url.URL // For parsing Url in templates +} + /* * Variables used by the upper ones */ @@ -186,14 +196,15 @@ type Navigation struct { type SearchForm struct { common.SearchParam - Category string - HideAdvancedSearch bool + Category string + ShowItemsPerPage bool } // Some Default Values to ease things out func NewSearchForm() SearchForm { return SearchForm{ Category: "_", + ShowItemsPerPage: true, } } diff --git a/router/userHandler.go b/router/userHandler.go index 5370896c..2017250a 100755 --- a/router/userHandler.go +++ b/router/userHandler.go @@ -66,9 +66,7 @@ func UserProfileHandler(w http.ResponseWriter, r *http.Request) { err["errors"] = append(err["errors"], errUser.Error()) } languages.SetTranslationFromRequest(viewUserDeleteTemplate, r, "en-us") - searchForm := NewSearchForm() - searchForm.HideAdvancedSearch = true - htv := UserVerifyTemplateVariables{err, searchForm, Navigation{}, GetUser(r), r.URL, mux.CurrentRoute(r)} + htv := UserVerifyTemplateVariables{err, NewSearchForm(), Navigation{}, GetUser(r), r.URL, mux.CurrentRoute(r)} errorTmpl := viewUserDeleteTemplate.ExecuteTemplate(w, "index.html", htv) if errorTmpl != nil { http.Error(w, errorTmpl.Error(), http.StatusInternalServerError) @@ -81,9 +79,7 @@ func UserProfileHandler(w http.ResponseWriter, r *http.Request) { if unfollow != nil { infosForm["infos"] = append(infosForm["infos"], fmt.Sprintf(T("user_unfollowed_msg"), userProfile.Username)) } - searchForm := NewSearchForm() - searchForm.HideAdvancedSearch = true - htv := UserProfileVariables{&userProfile, infosForm, searchForm, Navigation{}, currentUser, r.URL, mux.CurrentRoute(r)} + htv := UserProfileVariables{&userProfile, infosForm, NewSearchForm(), Navigation{}, currentUser, r.URL, mux.CurrentRoute(r)} err := viewProfileTemplate.ExecuteTemplate(w, "index.html", htv) if err != nil { @@ -91,11 +87,8 @@ func UserProfileHandler(w http.ResponseWriter, r *http.Request) { } } } else { - searchForm := NewSearchForm() - searchForm.HideAdvancedSearch = true - languages.SetTranslationFromRequest(notFoundTemplate, r, "en-us") - err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, searchForm, GetUser(r), r.URL, mux.CurrentRoute(r)}) + err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, NewSearchForm(), GetUser(r), r.URL, mux.CurrentRoute(r)}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -113,21 +106,16 @@ func UserDetailsHandler(w http.ResponseWriter, r *http.Request) { b := form.UserForm{} modelHelper.BindValueForm(&b, r) languages.SetTranslationFromRequest(viewProfileEditTemplate, r, "en-us") - searchForm := NewSearchForm() - searchForm.HideAdvancedSearch = true availableLanguages := languages.GetAvailableLanguages() - htv := UserProfileEditVariables{&userProfile, b, form.NewErrors(), form.NewInfos(), availableLanguages, searchForm, Navigation{}, currentUser, r.URL, mux.CurrentRoute(r)} + htv := UserProfileEditVariables{&userProfile, b, form.NewErrors(), form.NewInfos(), availableLanguages, NewSearchForm(), Navigation{}, currentUser, r.URL, mux.CurrentRoute(r)} err := viewProfileEditTemplate.ExecuteTemplate(w, "index.html", htv) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } } } else { - searchForm := NewSearchForm() - searchForm.HideAdvancedSearch = true - languages.SetTranslationFromRequest(notFoundTemplate, r, "en-us") - err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, searchForm, GetUser(r), r.URL, mux.CurrentRoute(r)}) + err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, NewSearchForm(), GetUser(r), r.URL, mux.CurrentRoute(r)}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -185,21 +173,15 @@ func UserProfileFormHandler(w http.ResponseWriter, r *http.Request) { http.Error(w, errorTmpl.Error(), http.StatusInternalServerError) } } else { - searchForm := NewSearchForm() - searchForm.HideAdvancedSearch = true - languages.SetTranslationFromRequest(notFoundTemplate, r, "en-us") - err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, searchForm, GetUser(r), r.URL, mux.CurrentRoute(r)}) + err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, NewSearchForm(), GetUser(r), r.URL, mux.CurrentRoute(r)}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } } } else { - searchForm := NewSearchForm() - searchForm.HideAdvancedSearch = true - languages.SetTranslationFromRequest(notFoundTemplate, r, "en-us") - err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, searchForm, GetUser(r), r.URL, mux.CurrentRoute(r)}) + err := notFoundTemplate.ExecuteTemplate(w, "index.html", NotFoundTemplateVariables{Navigation{}, NewSearchForm(), GetUser(r), r.URL, mux.CurrentRoute(r)}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/service/service.go b/service/service.go index 1d21b15c..772f2aee 100644 --- a/service/service.go +++ b/service/service.go @@ -5,7 +5,7 @@ type WhereParams struct { Params []interface{} } -func CreateWhereParams(conditions string, params ...string) WhereParams { +func CreateWhereParams(conditions string, params ...interface{}) WhereParams { whereParams := WhereParams{ Conditions: conditions, Params: make([]interface{}, len(params)), diff --git a/service/torrent/filesizeFetcher/filesizeFetcher.go b/service/torrent/filesizeFetcher/filesizeFetcher.go new file mode 100644 index 00000000..1f56f0c3 --- /dev/null +++ b/service/torrent/filesizeFetcher/filesizeFetcher.go @@ -0,0 +1,228 @@ +package filesizeFetcher; + +import ( + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/metainfo" + "github.com/ewhal/nyaa/config" + "github.com/ewhal/nyaa/db" + "github.com/ewhal/nyaa/model" + "github.com/ewhal/nyaa/util/log" + serviceBase "github.com/ewhal/nyaa/service" + torrentService "github.com/ewhal/nyaa/service/torrent" + "sync" + "time" +) + +type FilesizeFetcher struct { + torrentClient *torrent.Client + results chan Result + queueSize int + timeout int + maxDays int + done chan int + queue []*FetchOperation + queueMutex sync.Mutex + failedOperations map[uint]struct{} + wakeUp *time.Ticker + wg sync.WaitGroup +} + +func New(fetcherConfig *config.FilesizeFetcherConfig) (fetcher *FilesizeFetcher, err error) { + client, err := torrent.NewClient(nil) + fetcher = &FilesizeFetcher{ + torrentClient: client, + results: make(chan Result, fetcherConfig.QueueSize), + queueSize: fetcherConfig.QueueSize, + timeout: fetcherConfig.Timeout, + maxDays: fetcherConfig.MaxDays, + done: make(chan int, 1), + failedOperations: make(map[uint]struct{}), + wakeUp: time.NewTicker(time.Second * time.Duration(fetcherConfig.WakeUpInterval)), + } + + return +} + +func (fetcher *FilesizeFetcher) isFetchingOrFailed(t model.Torrent) bool { + for _, op := range fetcher.queue { + if op.torrent.ID == t.ID { + return true + } + } + + _, ok := fetcher.failedOperations[t.ID] + return ok +} + +func (fetcher *FilesizeFetcher) addToQueue(op *FetchOperation) bool { + fetcher.queueMutex.Lock() + defer fetcher.queueMutex.Unlock() + + if len(fetcher.queue) + 1 > fetcher.queueSize { + return false + } + + fetcher.queue = append(fetcher.queue, op) + return true +} + + +func (fetcher *FilesizeFetcher) removeFromQueue(op *FetchOperation) bool { + fetcher.queueMutex.Lock() + defer fetcher.queueMutex.Unlock() + + for i, queueOP := range fetcher.queue { + if queueOP == op { + fetcher.queue = append(fetcher.queue[:i], fetcher.queue[i+1:]...) + return true + } + } + + return false +} + +func updateFileList(dbEntry model.Torrent, info *metainfo.Info) error { + log.Infof("TID %d has %d files.", dbEntry.ID, len(info.Files)) + for _, file := range info.Files { + path := file.DisplayPath(info) + fileExists := false + for _, existingFile := range dbEntry.FileList { + if existingFile.Path == path { + fileExists = true + break + } + } + + if !fileExists { + log.Infof("Adding file %s to filelist of TID %d", path, dbEntry.ID) + dbFile := model.File{ + TorrentID: dbEntry.ID, + Path: path, + Filesize: file.Length, + } + + err := db.ORM.Create(&dbFile).Error + if err != nil { + return err + } + } + } + + return nil +} + +func (fetcher *FilesizeFetcher) gotResult(r Result) { + updatedSuccessfully := false + if r.err != nil { + log.Infof("Failed to get torrent filesize (TID: %d), err %v", r.operation.torrent.ID, r.err) + } else if r.info.TotalLength() == 0 { + log.Infof("Got length 0 for torrent TID: %d. Possible bug?", r.operation.torrent.ID) + } else { + log.Infof("Got length %d for torrent TID: %d. Updating.", r.info.TotalLength(), r.operation.torrent.ID) + r.operation.torrent.Filesize = r.info.TotalLength() + _, err := torrentService.UpdateTorrent(r.operation.torrent) + if err != nil { + log.Infof("Failed to update torrent TID: %d with new filesize", r.operation.torrent.ID) + } else { + updatedSuccessfully = true + } + + // Also update the File list with FileInfo, I guess. + err = updateFileList(r.operation.torrent, r.info) + if err != nil { + log.Infof("Failed to update file list of TID %d", r.operation.torrent.ID) + } + } + + if !updatedSuccessfully { + fetcher.failedOperations[r.operation.torrent.ID] = struct{}{} + } + + fetcher.removeFromQueue(r.operation) +} + +func (fetcher *FilesizeFetcher) fillQueue() { + toFill := fetcher.queueSize - len(fetcher.queue) + + if toFill <= 0 { + return + } + + oldest := time.Now().Add(0 - (time.Hour * time.Duration(24 * fetcher.maxDays))) + params := serviceBase.CreateWhereParams("(filesize IS NULL OR filesize = 0) AND date > ?", oldest) + // Get up to queueSize + len(failed) torrents, so we get at least some fresh new ones. + dbTorrents, count, err := torrentService.GetTorrents(params, fetcher.queueSize + len(fetcher.failedOperations), 0) + + if err != nil { + log.Infof("Failed to get torrents for filesize updating") + return + } + + if count == 0 { + log.Infof("No torrents for filesize update") + return + } + + for _, T := range dbTorrents { + if fetcher.isFetchingOrFailed(T) { + continue + } + + log.Infof("Added TID %d for filesize fetching", T.ID) + operation := NewFetchOperation(fetcher, T) + + if fetcher.addToQueue(operation) { + fetcher.wg.Add(1) + go operation.Start(fetcher.results) + } else { + break + } + } +} + +func (fetcher *FilesizeFetcher) run() { + var result Result + + defer fetcher.wg.Done() + + done := 0 + fetcher.fillQueue() + for done == 0 { + select { + case done = <-fetcher.done: + break + case result = <-fetcher.results: + fetcher.gotResult(result) + fetcher.fillQueue() + break + case <-fetcher.wakeUp.C: + fetcher.fillQueue() + break + } + } +} + +func (fetcher *FilesizeFetcher) RunAsync() { + fetcher.wg.Add(1) + + go fetcher.run() +} + +func (fetcher *FilesizeFetcher) Close() error { + fetcher.queueMutex.Lock() + defer fetcher.queueMutex.Unlock() + + // Send the done event to every Operation + for _, op := range fetcher.queue { + op.done <- 1 + } + + fetcher.done <- 1 + fetcher.wg.Wait() + return nil +} + +func (fetcher *FilesizeFetcher) Wait() { + fetcher.wg.Wait() +} + diff --git a/service/torrent/filesizeFetcher/operation.go b/service/torrent/filesizeFetcher/operation.go new file mode 100644 index 00000000..1a08d061 --- /dev/null +++ b/service/torrent/filesizeFetcher/operation.go @@ -0,0 +1,60 @@ +package filesizeFetcher; + +import ( + "github.com/anacrolix/torrent/metainfo" + "github.com/ewhal/nyaa/config" + "github.com/ewhal/nyaa/model" + "github.com/ewhal/nyaa/util" + "errors" + "time" + "strings" +) + +type FetchOperation struct { + fetcher *FilesizeFetcher + torrent model.Torrent + done chan int +} + +type Result struct { + operation *FetchOperation + err error + info *metainfo.Info +} + +func NewFetchOperation(fetcher *FilesizeFetcher, dbEntry model.Torrent) (op *FetchOperation) { + op = &FetchOperation{ + fetcher: fetcher, + torrent: dbEntry, + done: make(chan int, 1), + } + return +} + +// Should be started from a goroutine somewhere +func (op *FetchOperation) Start(out chan Result) { + defer op.fetcher.wg.Done() + + magnet := util.InfoHashToMagnet(strings.TrimSpace(op.torrent.Hash), op.torrent.Name, config.Trackers...) + downloadingTorrent, err := op.fetcher.torrentClient.AddMagnet(magnet) + if err != nil { + out <- Result{op, err, nil} + return + } + + timeoutTicker := time.NewTicker(time.Second * time.Duration(op.fetcher.timeout)) + select { + case <-downloadingTorrent.GotInfo(): + downloadingTorrent.Drop() + out <- Result{op, nil, downloadingTorrent.Info()} + break + case <-timeoutTicker.C: + downloadingTorrent.Drop() + out <- Result{op, errors.New("Timeout"), nil} + break + case <-op.done: + downloadingTorrent.Drop() + break + } +} + diff --git a/service/torrent/filesizeFetcher/operation_test.go b/service/torrent/filesizeFetcher/operation_test.go new file mode 100644 index 00000000..21e36faa --- /dev/null +++ b/service/torrent/filesizeFetcher/operation_test.go @@ -0,0 +1,45 @@ +package filesizeFetcher; + +import ( + "testing" + + "github.com/anacrolix/torrent" + "github.com/ewhal/nyaa/model" +) + +func TestInvalidHash(t *testing.T) { + client, err := torrent.NewClient(nil) + if err != nil { + t.Skipf("Failed to create client, with err %v. Skipping.", err) + } + + fetcher := &FilesizeFetcher{ + timeout: 5, + torrentClient: client, + results: make(chan Result, 1), + } + + dbEntry := model.Torrent{ + Hash: "INVALID", + Name: "Invalid", + } + + op := NewFetchOperation(fetcher, dbEntry) + fetcher.wg.Add(1) + op.Start(fetcher.results) + + var res Result + select { + case res = <-fetcher.results: + break + default: + t.Fatal("No result in channel, should have one") + } + + if res.err == nil { + t.Fatal("Got no error, should have got invalid magnet") + } + + t.Logf("Got error %s, shouldn't be timeout", res.err) +} + diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go index 7c5eade4..cadfff5f 100644 --- a/service/torrent/torrent.go +++ b/service/torrent/torrent.go @@ -71,7 +71,7 @@ func GetTorrentById(id string) (torrent model.Torrent, err error) { torrent.Uploader = new(model.User) db.ORM.Where("user_id = ?", torrent.UploaderID).Find(torrent.Uploader) torrent.OldUploader = "" - if torrent.ID <= config.LastOldTorrentID { + if torrent.ID <= config.LastOldTorrentID && torrent.UploaderID == 0 { var tmp model.UserUploadsOld if !db.ORM.Where("torrent_id = ?", torrent.ID).Find(&tmp).RecordNotFound() { torrent.OldUploader = tmp.Username @@ -88,6 +88,15 @@ func GetTorrentById(id string) (torrent model.Torrent, err error) { return } +// won't fetch user or comments +func GetRawTorrentById(id uint) (torrent model.Torrent, err error) { + err = nil + if db.ORM.Where("torrent_id = ?", id).Find(&torrent).RecordNotFound() { + err = errors.New("Article is not found.") + } + return +} + func GetTorrentsOrderByNoCount(parameters *serviceBase.WhereParams, orderBy string, limit int, offset int) (torrents []model.Torrent, err error) { torrents, _, err = getTorrentsOrderBy(parameters, orderBy, limit, offset, false) return diff --git a/service/upload/upload.go b/service/upload/upload.go index 8335db1a..8bcd309b 100644 --- a/service/upload/upload.go +++ b/service/upload/upload.go @@ -16,7 +16,11 @@ func CheckTrackers(trackers []string) bool { "://tracker.istole.it:80", "://tracker.ccc.de:80", "://bt2.careland.com.cn:6969", - "://announce.torrentsmd.com:8080"} + "://announce.torrentsmd.com:8080", + "://open.demonii.com:1337", + "://tracker.btcake.com", + "://tracker.prq.to", + "://bt.rghost.net"} var numGood int for _, t := range trackers { diff --git a/service/user/form/formValidator.go b/service/user/form/formValidator.go index 10218f64..5ee026b1 100644 --- a/service/user/form/formValidator.go +++ b/service/user/form/formValidator.go @@ -50,7 +50,7 @@ func IsAgreed(termsAndConditions string) bool { // TODO: Inline function type RegistrationForm struct { Username string `form:"username" needed:"true" len_min:"3" len_max:"20"` Email string `form:"email"` - Password string `form:"password" needed:"true" len_min:"6" len_max:"25" equalInput:"ConfirmPassword"` + Password string `form:"password" needed:"true" len_min:"6" len_max:"72" equalInput:"ConfirmPassword"` ConfirmPassword string `form:"password_confirmation" omit:"true" needed:"true"` CaptchaID string `form:"captchaID" omit:"true" needed:"true"` TermsAndConditions bool `form:"t_and_c" omit:"true" needed:"true" equal:"true" hum_name:"Terms and Conditions"` @@ -67,8 +67,8 @@ type UserForm struct { Username string `form:"username" needed:"true" len_min:"3" len_max:"20"` Email string `form:"email"` Language string `form:"language" default:"en-us"` - CurrentPassword string `form:"current_password" len_min:"6" len_max:"25" omit:"true"` - Password string `form:"password" len_min:"6" len_max:"25" equalInput:"Confirm_Password"` + CurrentPassword string `form:"current_password" len_min:"6" len_max:"72" omit:"true"` + Password string `form:"password" len_min:"6" len_max:"72" equalInput:"Confirm_Password"` Confirm_Password string `form:"password_confirmation" omit:"true"` Status int `form:"status" default:"0"` } diff --git a/service/user/user.go b/service/user/user.go index ece49467..634481b2 100644 --- a/service/user/user.go +++ b/service/user/user.go @@ -253,6 +253,19 @@ func RetrieveUserByUsername(username string) (*model.PublicUser, string, int, er return &model.PublicUser{User: &user}, username, http.StatusOK, nil } +func RetrieveOldUploadsByUsername(username string) ([]uint, error) { + var ret []uint + var tmp []*model.UserUploadsOld + err := db.ORM.Where("username = ?", username).Find(&tmp).Error + if err != nil { + return ret, err + } + for _, tmp2 := range tmp { + ret = append(ret, tmp2.TorrentId) + } + return ret, nil +} + // RetrieveUserForAdmin retrieves a user for an administrator. func RetrieveUserForAdmin(id string) (model.User, int, error) { var user model.User diff --git a/templates/FAQ.html b/templates/FAQ.html index a29d19c4..28703d9e 100644 --- a/templates/FAQ.html +++ b/templates/FAQ.html @@ -39,14 +39,15 @@

{{T "which_trackers_do_you_recommend"}}

{{T "answer_which_trackers_do_you_recommend"}}

- 
udp://tracker.coppersurfer.tk:6969
-udp://zer0day.to:1337/announce
-udp://tracker.leechers-paradise.org:6969
-udp://explodie.org:6969
-udp://tracker.opentrackr.org:1337
-udp://tracker.internetwarriors.net:1337/announce
-http://mgtracker.org:6969/announce
-http://tracker.baka-sub.cf/announce
+ 
udp://tracker.doko.moe:6969
+

{{T "other_trackers"}}

+ 
udp://zer0day.to:1337/announce
+             udp://tracker.leechers-paradise.org:6969
+             udp://explodie.org:6969
+             udp://tracker.opentrackr.org:1337
+             udp://tracker.internetwarriors.net:1337/announce
+            http://mgtracker.org:6969/announce
+            http://tracker.baka-sub.cf/announce

{{T "how_can_i_help"}}

{{T "answer_how_can_i_help"}}

diff --git a/templates/_badgemenu.html b/templates/_badgemenu.html index 136f6ee3..bf3d25aa 100644 --- a/templates/_badgemenu.html +++ b/templates/_badgemenu.html @@ -7,6 +7,7 @@ {{ .Username }} {{block "badge_user" .}}{{end}} diff --git a/templates/home.html b/templates/home.html index ae895757..037f3af0 100644 --- a/templates/home.html +++ b/templates/home.html @@ -1,7 +1,7 @@ {{define "title"}}{{T "home"}}{{end}} {{define "contclass"}}cont-home{{end}} {{define "content"}} -
+