From 33fe211d3f656793f3dbad5be8a0f6bb2f31747f Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 11 May 2017 06:49:05 -0400 Subject: [PATCH 01/27] mod panel template cleanup, make mod panel templates reload on SIGHUP --- router/modpanel.go | 31 ++++-------------- router/template.go | 79 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 70 insertions(+), 40 deletions(-) diff --git a/router/modpanel.go b/router/modpanel.go index d810282a..834c28d1 100644 --- a/router/modpanel.go +++ b/router/modpanel.go @@ -3,9 +3,7 @@ package router import ( "fmt" "html" - "html/template" "net/http" - "path/filepath" "strconv" "github.com/ewhal/nyaa/db" @@ -23,23 +21,6 @@ import ( "github.com/gorilla/mux" ) -var panelIndex, panelTorrentList, panelUserList, panelCommentList, panelTorrentEd, panelTorrentReportList *template.Template - -func init() { - panelTorrentList = template.Must(template.New("torrentlist").Funcs(FuncMap).ParseFiles(filepath.Join(TemplateDir, "admin_index.html"), filepath.Join(TemplateDir, "admin/torrentlist.html"))) - panelTorrentList = template.Must(panelTorrentList.ParseGlob(filepath.Join(TemplateDir, "_*.html"))) - panelUserList = template.Must(template.New("userlist").Funcs(FuncMap).ParseFiles(filepath.Join(TemplateDir, "admin_index.html"), filepath.Join(TemplateDir, "admin/userlist.html"))) - panelUserList = template.Must(panelUserList.ParseGlob(filepath.Join(TemplateDir, "_*.html"))) - panelCommentList = template.Must(template.New("commentlist").Funcs(FuncMap).ParseFiles(filepath.Join(TemplateDir, "admin_index.html"), filepath.Join(TemplateDir, "admin/commentlist.html"))) - panelCommentList = template.Must(panelCommentList.ParseGlob(filepath.Join(TemplateDir, "_*.html"))) - panelIndex = template.Must(template.New("indexPanel").Funcs(FuncMap).ParseFiles(filepath.Join(TemplateDir, "admin_index.html"), filepath.Join(TemplateDir, "admin/panelindex.html"))) - panelIndex = template.Must(panelIndex.ParseGlob(filepath.Join(TemplateDir, "_*.html"))) - panelTorrentEd = template.Must(template.New("torrent_ed").Funcs(FuncMap).ParseFiles(filepath.Join(TemplateDir, "admin_index.html"), filepath.Join(TemplateDir, "admin/paneltorrentedit.html"))) - panelTorrentEd = template.Must(panelTorrentEd.ParseGlob(filepath.Join(TemplateDir, "_*.html"))) - panelTorrentReportList = template.Must(template.New("torrent_report").Funcs(FuncMap).ParseFiles(filepath.Join(TemplateDir, "admin_index.html"), filepath.Join(TemplateDir, "admin/torrent_report.html"))) - panelTorrentReportList = template.Must(panelTorrentReportList.ParseGlob(filepath.Join(TemplateDir, "_*.html"))) -} - func IndexModPanel(w http.ResponseWriter, r *http.Request) { currentUser := GetUser(r) if userPermission.HasAdmin(currentUser) { @@ -217,19 +198,19 @@ func TorrentPostEditModPanel(w http.ResponseWriter, r *http.Request) { err := form.NewErrors() infos := form.NewInfos() torrent, _ := torrentService.GetTorrentById(id) - if (torrent.ID > 0) { + if torrent.ID > 0 { errUp := uploadForm.ExtractEditInfo(r) if errUp != nil { err["errors"] = append(err["errors"], "Failed to update torrent!") } - if (len(err) == 0) { + if len(err) == 0 { // update some (but not all!) values - torrent.Name = uploadForm.Name - torrent.Category = uploadForm.CategoryID + torrent.Name = uploadForm.Name + torrent.Category = uploadForm.CategoryID torrent.SubCategory = uploadForm.SubCategoryID - torrent.Status = uploadForm.Status + torrent.Status = uploadForm.Status torrent.Description = uploadForm.Description - torrent.Uploader = nil // GORM will create a new user otherwise (wtf?!) + torrent.Uploader = nil // GORM will create a new user otherwise (wtf?!) db.ORM.Save(&torrent) infos["infos"] = append(infos["infos"], "Torrent details updated.") } diff --git a/router/template.go b/router/template.go index edf9ac20..216715d4 100644 --- a/router/template.go +++ b/router/template.go @@ -5,19 +5,22 @@ import ( "path/filepath" ) -var TemplateDir = "templates" +const TemplateDir = "templates" var homeTemplate, searchTemplate, faqTemplate, uploadTemplate, viewTemplate, viewRegisterTemplate, viewLoginTemplate, viewRegisterSuccessTemplate, viewVerifySuccessTemplate, viewProfileTemplate, viewProfileEditTemplate, viewUserDeleteTemplate, notFoundTemplate *template.Template +var panelIndex, panelTorrentList, panelUserList, panelCommentList, panelTorrentEd, panelTorrentReportList *template.Template + type templateLoader struct { - templ **template.Template - file string - name string + templ **template.Template + file string + indexFile string + name string } // ReloadTemplates reloads templates on runtime func ReloadTemplates() { - templs := []templateLoader{ + pubTempls := []templateLoader{ templateLoader{ templ: &homeTemplate, name: "home", @@ -46,37 +49,37 @@ func ReloadTemplates() { templateLoader{ templ: &viewRegisterTemplate, name: "user_register", - file: "user/register.html", + file: filepath.Join("user", "register.html"), }, templateLoader{ templ: &viewRegisterSuccessTemplate, name: "user_register_success", - file: "user/signup_success.html", + file: filepath.Join("user", "signup_success.html"), }, templateLoader{ templ: &viewVerifySuccessTemplate, name: "user_verify_success", - file: "user/verify_success.html", + file: filepath.Join("user", "verify_success.html"), }, templateLoader{ templ: &viewLoginTemplate, name: "user_login", - file: "user/login.html", + file: filepath.Join("user", "login.html"), }, templateLoader{ templ: &viewProfileTemplate, name: "user_profile", - file: "user/profile.html", + file: filepath.Join("user", "profile.html"), }, templateLoader{ templ: &viewProfileEditTemplate, name: "user_profile", - file: "user/profile_edit.html", + file: filepath.Join("user", "profile_edit.html"), }, templateLoader{ templ: &viewUserDeleteTemplate, name: "user_delete", - file: "user/delete_success.html", + file: filepath.Join("user", "delete_success.html"), }, templateLoader{ templ: ¬FoundTemplate, @@ -84,10 +87,56 @@ func ReloadTemplates() { file: "404.html", }, } - for _, templ := range templs { - t := template.Must(template.New(templ.name).Funcs(FuncMap).ParseFiles(filepath.Join(TemplateDir, "index.html"), filepath.Join(TemplateDir, templ.file))) - t = template.Must(t.ParseGlob(filepath.Join(TemplateDir, "_*.html"))) + for idx := range pubTempls { + pubTempls[idx].indexFile = filepath.Join(TemplateDir, "index.html") + } + modTempls := []templateLoader{ + templateLoader{ + templ: &panelTorrentList, + name: "torrentlist", + file: filepath.Join("admin", "torrentlist.html"), + }, + templateLoader{ + templ: &panelUserList, + name: "userlist", + file: filepath.Join("admin", "userlist.html"), + }, + templateLoader{ + templ: &panelCommentList, + name: "commentlist", + file: filepath.Join("admin", "commentlist.html"), + }, + templateLoader{ + templ: &panelIndex, + name: "indexPanel", + file: filepath.Join("admin", "panelindex.html"), + }, + templateLoader{ + templ: &panelTorrentEd, + name: "torrent_ed", + file: filepath.Join("admin", "paneltorrentedit.html"), + }, + templateLoader{ + templ: &panelTorrentReportList, + name: "torrent_report", + file: filepath.Join("admin", "torrent_report.html"), + }, + } + + for idx := range modTempls { + modTempls[idx].indexFile = filepath.Join(TemplateDir, "admin_index.html") + } + + templs := make([]templateLoader, 0, len(modTempls)+len(pubTempls)) + + templs = append(templs, pubTempls...) + templs = append(templs, modTempls...) + + for _, templ := range templs { + t := template.Must(template.New(templ.name).Funcs(FuncMap).ParseFiles(templ.indexFile, filepath.Join(TemplateDir, templ.file))) + t = template.Must(t.ParseGlob(filepath.Join(TemplateDir, "_*.html"))) *templ.templ = t } + } From 57895251a3317e68ce03d84ae6d9e58a19525431 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 11 May 2017 07:40:50 -0400 Subject: [PATCH 02/27] fix torrent swarm ordering and add timeouts for udp scrape --- common/search.go | 4 +++ main.go | 1 + service/scraper/bucket.go | 29 +++++++++++++++++- service/scraper/scraper.go | 54 ++++++++++++++++++++++++++-------- service/scraper/transaction.go | 16 +++++++--- util/search/search.go | 20 +++++++++++-- 6 files changed, 104 insertions(+), 20 deletions(-) diff --git a/common/search.go b/common/search.go index c9679e79..eace727f 100644 --- a/common/search.go +++ b/common/search.go @@ -19,6 +19,9 @@ const ( Date Downloads Size + Seeders + Leechers + Completed ) type Category struct { @@ -44,5 +47,6 @@ type SearchParam struct { Page int UserID uint Max uint + NotNull string Query string } diff --git a/main.go b/main.go index 815b7579..8c92a00c 100644 --- a/main.go +++ b/main.go @@ -84,6 +84,7 @@ func RunScraper(conf *config.Config) { signals.RegisterCloser(scraper) // run udp scraper worker for workers > 0 { + log.Infof("starting up worker %d", workers) go scraper.RunWorker(pc) workers-- } diff --git a/service/scraper/bucket.go b/service/scraper/bucket.go index da301528..d9c85372 100644 --- a/service/scraper/bucket.go +++ b/service/scraper/bucket.go @@ -28,15 +28,42 @@ func (b *Bucket) NewTransaction(swarms []model.Torrent) (t *Transaction) { t = &Transaction{ TransactionID: id, bucket: b, - swarms: swarms, + swarms: make([]model.Torrent, len(swarms)), state: stateSendID, } + copy(t.swarms, swarms) b.transactions[id] = t b.access.Unlock() return } +func (b *Bucket) ForEachTransaction(v func(uint32, *Transaction)) { + + clone := make(map[uint32]*Transaction) + + b.access.Lock() + + for k := range b.transactions { + clone[k] = b.transactions[k] + } + + b.access.Unlock() + + for k := range clone { + v(k, clone[k]) + } +} + +func (b *Bucket) Forget(tid uint32) { + b.access.Lock() + _, ok := b.transactions[tid] + if ok { + delete(b.transactions, tid) + } + b.access.Unlock() +} + func (b *Bucket) VisitTransaction(tid uint32, v func(*Transaction)) { b.access.Lock() t, ok := b.transactions[tid] diff --git a/service/scraper/scraper.go b/service/scraper/scraper.go index 8077f41b..7d4b8e61 100644 --- a/service/scraper/scraper.go +++ b/service/scraper/scraper.go @@ -13,15 +13,20 @@ import ( // MTU yes this is the ipv6 mtu const MTU = 1500 +// max number of scrapes per packet +const ScrapesPerPacket = 74 + // bittorrent scraper type Scraper struct { - done chan int - sendQueue chan *SendEvent - recvQueue chan *RecvEvent - errQueue chan error - trackers map[string]*Bucket - ticker *time.Ticker - interval time.Duration + done chan int + sendQueue chan *SendEvent + recvQueue chan *RecvEvent + errQueue chan error + trackers map[string]*Bucket + ticker *time.Ticker + cleanup *time.Ticker + interval time.Duration + PacketsPerSecond uint } func New(conf *config.ScraperConfig) (sc *Scraper, err error) { @@ -33,7 +38,13 @@ func New(conf *config.ScraperConfig) (sc *Scraper, err error) { trackers: make(map[string]*Bucket), ticker: time.NewTicker(time.Second), interval: time.Second * time.Duration(conf.IntervalSeconds), + cleanup: time.NewTicker(time.Second), } + + if sc.PacketsPerSecond == 0 { + sc.PacketsPerSecond = 10 + } + for idx := range conf.Trackers { err = sc.AddTracker(&conf.Trackers[idx]) if err != nil { @@ -144,20 +155,37 @@ func (sc *Scraper) RunWorker(pc net.PacketConn) (err error) { func (sc *Scraper) Run() { for { - <-sc.ticker.C - sc.Scrape() + select { + case <-sc.ticker.C: + sc.Scrape(sc.PacketsPerSecond) + break + case <-sc.cleanup.C: + sc.removeStale() + break + } } } -func (sc *Scraper) Scrape() { +func (sc *Scraper) removeStale() { + + for k := range sc.trackers { + sc.trackers[k].ForEachTransaction(func(tid uint32, t *Transaction) { + if t == nil || t.IsTimedOut() { + sc.trackers[k].Forget(tid) + } + }) + } +} + +func (sc *Scraper) Scrape(packets uint) { now := time.Now().Add(0 - sc.interval) - rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM torrents WHERE last_scrape IS NULL OR last_scrape < ? ORDER BY torrent_id DESC LIMIT 700", now).Rows() + rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM torrents WHERE last_scrape IS NULL OR last_scrape < ? ORDER BY torrent_id DESC LIMIT ?", now, packets*ScrapesPerPacket).Rows() if err == nil { counter := 0 - var scrape [70]model.Torrent + var scrape [ScrapesPerPacket]model.Torrent for rows.Next() { - idx := counter % 70 + idx := counter % ScrapesPerPacket rows.Scan(&scrape[idx].ID, &scrape[idx].Hash) counter++ if idx == 0 { diff --git a/service/scraper/transaction.go b/service/scraper/transaction.go index 90ebcf77..7b7de953 100644 --- a/service/scraper/transaction.go +++ b/service/scraper/transaction.go @@ -11,6 +11,9 @@ import ( "github.com/ewhal/nyaa/util/log" ) +// TransactionTimeout 30 second timeout for transactions +const TransactionTimeout = time.Second * 30 + const stateSendID = 0 const stateRecvID = 1 const stateTransact = 2 @@ -27,13 +30,12 @@ type Transaction struct { bucket *Bucket state uint8 swarms []model.Torrent + lastData time.Time } // Done marks this transaction as done and removes it from parent func (t *Transaction) Done() { - t.bucket.access.Lock() - delete(t.bucket.transactions, t.TransactionID) - t.bucket.access.Unlock() + t.bucket.Forget(t.TransactionID) } func (t *Transaction) handleScrapeReply(data []byte) { @@ -95,6 +97,7 @@ func (t *Transaction) SendEvent(to net.Addr) (ev *SendEvent) { binary.BigEndian.PutUint32(ev.Data[12:], t.TransactionID) t.state = stateRecvID } + t.lastData = time.Now() return } @@ -104,7 +107,7 @@ func (t *Transaction) handleError(msg string) { // handle data for transaction func (t *Transaction) GotData(data []byte) (done bool) { - + t.lastData = time.Now() if len(data) > 4 { cmd := binary.BigEndian.Uint32(data) switch cmd { @@ -132,3 +135,8 @@ func (t *Transaction) GotData(data []byte) (done bool) { } return } + +func (t *Transaction) IsTimedOut() bool { + return t.lastData.Add(TransactionTimeout).Before(time.Now()) + +} diff --git a/util/search/search.go b/util/search/search.go index a2e7ec47..ba66540f 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -40,7 +40,7 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( search.Page = pagenum search.Query = r.URL.Query().Get("q") userID, _ := strconv.Atoi(r.URL.Query().Get("userID")) - search.UserID = uint(userID) + search.UserID = uint(userID) switch s := r.URL.Query().Get("s"); s { case "1": @@ -75,22 +75,36 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( case "1": search.Sort = common.Name orderBy += "torrent_name" + break case "2": search.Sort = common.Date orderBy += "date" + break case "3": search.Sort = common.Downloads orderBy += "downloads" + break case "4": search.Sort = common.Size orderBy += "filesize" + break case "5": + search.Sort = common.Seeders orderBy += "seeders" + search.NotNull += "seeders IS NOT NULL " + break case "6": + search.Sort = common.Leechers orderBy += "leechers" + search.NotNull += "leechers IS NOT NULL " + break case "7": + search.Sort = common.Completed orderBy += "completed" + search.NotNull += "completed IS NOT NULL " + break default: + search.Sort = common.ID orderBy += "torrent_id" } @@ -129,7 +143,9 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( } parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1)) } - + if len(search.NotNull) > 0 { + conditions = append(conditions, search.NotNull) + } searchQuerySplit := strings.Fields(search.Query) for i, word := range searchQuerySplit { firstRune, _ := utf8.DecodeRuneInString(word) From a361bcca4f4886afb14c034b036a079aaba65a65 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 11 May 2017 07:42:26 -0400 Subject: [PATCH 03/27] fix template tests --- router/template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/template.go b/router/template.go index 216715d4..26314ea4 100644 --- a/router/template.go +++ b/router/template.go @@ -5,7 +5,7 @@ import ( "path/filepath" ) -const TemplateDir = "templates" +var TemplateDir = "templates" var homeTemplate, searchTemplate, faqTemplate, uploadTemplate, viewTemplate, viewRegisterTemplate, viewLoginTemplate, viewRegisterSuccessTemplate, viewVerifySuccessTemplate, viewProfileTemplate, viewProfileEditTemplate, viewUserDeleteTemplate, notFoundTemplate *template.Template From 25675745435ac7a2f9e2694c5c44a8761b816733 Mon Sep 17 00:00:00 2001 From: MMP0 Date: Thu, 11 May 2017 21:10:41 +0900 Subject: [PATCH 04/27] Update Japanese translation --- translations/ja-jp.all.json | 60 ++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/translations/ja-jp.all.json b/translations/ja-jp.all.json index 415f7000..d026244c 100644 --- a/translations/ja-jp.all.json +++ b/translations/ja-jp.all.json @@ -45,15 +45,15 @@ }, { "id":"confirm_password", - "translation": "パスワードを確認する" + "translation": "パスワードの再入力" }, { "id":"i_agree", - "translation": "同意します" + "translation": "同意する" }, { "id":"terms_conditions_confirm", - "translation": "登録する をクリックすることにより、Cookie の使用を含む、本サイトで規定されている 利用規約 に同意するものとします。" + "translation": "登録 をクリックすることにより、Cookie の使用を含む、本サイトの 利用規約 に同意したものとみなします。" }, { "id":"signin", @@ -213,7 +213,7 @@ }, { "id": "notice_keep_seeding", - "translation": "お願い：DHT 機能を有効にし、なるべくシードを継続してください" + "translation": "お願い: DHT 機能を有効にし、なるべくシードを継続してください" }, { "id": "official_nyaapocalipse_faq", @@ -289,7 +289,7 @@ }, { "id": "magnet_link_should_look_like", - "translation": "magnet リンクは次のようになっているはずです：" + "translation": "magnet リンクは次のようになっているはずです:" }, { "id": "which_trackers_do_you_recommend", @@ -297,7 +297,7 @@ }, { "id": "answer_which_trackers_do_you_recommend", - "translation": "トラッカーに Torrent のアップロードを拒否された場合は、この中のいくつかを追加する必要があります：" + "translation": "トラッカーに Torrent のアップロードを拒否された場合は、この中のいくつかを追加する必要があります:" }, { "id": "how_can_i_help", @@ -321,7 +321,7 @@ }, { "id": "nyaa_pantsu_dont_host_files", - "translation": " nyaa.pantsu.cat と sukebei.pantsu.cat はいかなるファイルもホストしていません。" + "translation": "nyaa.pantsu.cat と sukebei.pantsu.cat はいかなるファイルもホストしていません。" }, { "id": "upload_magnet", @@ -449,7 +449,7 @@ }, { "id": "filter_remakes", - "translation": "リメイクされたフィルター" + "translation": "再構成されたフィルター" }, { "id": "trusted", @@ -550,5 +550,49 @@ { "id": "delete_success", "translation": "アカウントが削除されました。" + }, + { + "id": "moderation", + "translation": "緩和" + }, + { + "id": "who_is_renchon", + "translation": "「れんちょん」って誰やねん" + }, + { + "id": "renchon_anon_explanation", + "translation": "「れんちょん」は匿名でアップロードもしくはコメントを書き込んだ際に割り当てられるユーザー名なのん。オリジナルのアップロード者と一緒に表示されることがあるけど、オリジナルの nyaa からインポートされた Torrent にも使用されるのん。" + }, + { + "id": "mark_as_remake", + "translation": "再構成としてマーク" + }, + { + "id": "email_changed", + "translation": "メールアドレスが変更されました。なお、送信されたリンクをクリックして認証を済ませる必要があります: %s" + }, + { + "id": "torrent_status", + "translation": "Torrent の状態" + }, + { + "id": "torrent_status_hidden", + "translation": "非表示" + }, + { + "id": "torrent_status_normal", + "translation": "通常" + }, + { + "id": "torrent_status_remake", + "translation": "再構成" + }, + { + "id": "profile_edit_page", + "translation": "%s のプロフィールを編集" + }, + { + "id":"date_format", + "translation": "2006/01/02 15:04" } ] From c1cafb9d9a2ce9ce1ddfb0ac58ecf3bd9b15e016 Mon Sep 17 00:00:00 2001 From: Eliot Whalan Date: Thu, 11 May 2017 22:22:49 +1000 Subject: [PATCH 05/27] Remove broken caching stuff --- cache/cache.go | 132 ------------------------------------------ main.go | 2 - router/homeHandler.go | 15 +---- router/upload.go | 4 +- util/search/search.go | 117 ++++++++++++++++++------------------- 5 files changed, 60 insertions(+), 210 deletions(-) delete mode 100644 cache/cache.go diff --git a/cache/cache.go b/cache/cache.go deleted file mode 100644 index b07f2d40..00000000 --- a/cache/cache.go +++ /dev/null @@ -1,132 +0,0 @@ -package cache - -import ( - "container/list" - "sync" - "time" - - "github.com/ewhal/nyaa/common" - "github.com/ewhal/nyaa/model" -) - -const expiryTime = time.Minute - -var ( - cache = make(map[common.SearchParam]*list.Element, 10) - ll = list.New() - totalUsed int - mu sync.Mutex - - // Size sets the maximum size of the cache before evicting unread data in MB - Size float64 = 1 << 10 -) - -// Key stores the ID of either a thread or board page -type Key struct { - LastN uint8 - Board string - ID uint64 -} - -// Single cache entry -type store struct { - sync.Mutex // Controls general access to the contents of the struct - lastFetched time.Time - key common.SearchParam - data []model.Torrent - count, size int -} - -// Check the cache for and existing record. If miss, run fn to retrieve fresh -// values. -func Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) ( - data []model.Torrent, count int, err error, -) { - s := getStore(key) - - // Also keeps multiple requesters from simultaneously requesting the same - // data - s.Lock() - defer s.Unlock() - - if s.isFresh() { - return s.data, s.count, nil - } - - data, count, err = fn() - if err != nil { - return - } - s.update(data, count) - return -} - -// Retrieve a store from the cache or create a new one -func getStore(k common.SearchParam) (s *store) { - mu.Lock() - defer mu.Unlock() - - el := cache[k] - if el == nil { - s = &store{key: k} - cache[k] = ll.PushFront(s) - } else { - ll.MoveToFront(el) - s = el.Value.(*store) - } - return s -} - -// Clear the cache. Only used for testing. -func Clear() { - mu.Lock() - defer mu.Unlock() - - ll = list.New() - cache = make(map[common.SearchParam]*list.Element, 10) -} - -// Update the total used memory counter and evict, if over limit -func updateUsedSize(delta int) { - mu.Lock() - defer mu.Unlock() - - totalUsed += delta - - for totalUsed > int(Size)<<20 { - e := ll.Back() - if e == nil { - break - } - s := ll.Remove(e).(*store) - delete(cache, s.key) - totalUsed -= s.size - } -} - -// Return, if the data can still be considered fresh, without querying the DB -func (s *store) isFresh() bool { - if s.lastFetched.IsZero() { // New store - return false - } - return s.lastFetched.Add(expiryTime).After(time.Now()) -} - -// Stores the new values of s. Calculates and stores the new size. Passes the -// delta to the central cache to fire eviction checks. -func (s *store) update(data []model.Torrent, count int) { - newSize := 0 - for _, d := range data { - newSize += d.Size() - } - s.data = data - s.count = count - delta := newSize - s.size - s.size = newSize - s.lastFetched = time.Now() - - // Technically it is possible to update the size even when the store is - // already evicted, but that should never happen, unless you have a very - // small cache, very large stored datasets and a lot of traffic. - updateUsedSize(delta) -} diff --git a/main.go b/main.go index 8c92a00c..700b7a47 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,6 @@ import ( "path/filepath" "time" - "github.com/ewhal/nyaa/cache" "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/network" @@ -98,7 +97,6 @@ func main() { processFlags := conf.BindFlags() defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout") mode := flag.String("mode", "webapp", "which mode to run daemon in, either webapp or scraper") - flag.Float64Var(&cache.Size, "c", cache.Size, "size of the search cache in MB") flag.Parse() if *defaults { stdout := bufio.NewWriter(os.Stdout) diff --git a/router/homeHandler.go b/router/homeHandler.go index ef2156e0..0735515b 100644 --- a/router/homeHandler.go +++ b/router/homeHandler.go @@ -5,8 +5,6 @@ import ( "net/http" "strconv" - "github.com/ewhal/nyaa/cache" - "github.com/ewhal/nyaa/common" "github.com/ewhal/nyaa/model" "github.com/ewhal/nyaa/service/torrent" "github.com/ewhal/nyaa/util" @@ -39,17 +37,10 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) { } } - search := common.SearchParam{ - Max: uint(maxPerPage), - Page: pagenum, + torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) + if !log.CheckError(err) { + util.SendError(w, err, 400) } - torrents, nbTorrents, err := cache.Get(search, func() ([]model.Torrent, int, error) { - torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) - if !log.CheckError(err) { - util.SendError(w, err, 400) - } - return torrents, nbTorrents, err - }) b := model.TorrentsToJSON(torrents) diff --git a/router/upload.go b/router/upload.go index 1eeb3b5c..1419dc63 100644 --- a/router/upload.go +++ b/router/upload.go @@ -14,7 +14,6 @@ import ( "strconv" "strings" - "github.com/ewhal/nyaa/cache" "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/service/captcha" "github.com/ewhal/nyaa/util" @@ -30,7 +29,7 @@ type UploadForm struct { Category string Remake bool Description string - Status int + Status int captcha.Captcha Infohash string @@ -93,7 +92,6 @@ func (f *UploadForm) ExtractInfo(r *http.Request) error { f.Name = util.TrimWhitespaces(f.Name) f.Description = p.Sanitize(util.TrimWhitespaces(f.Description)) f.Magnet = util.TrimWhitespaces(f.Magnet) - cache.Clear() catsSplit := strings.Split(f.Category, "_") // need this to prevent out of index panics diff --git a/util/search/search.go b/util/search/search.go index ba66540f..a30a08ca 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -7,7 +7,6 @@ import ( "unicode" "unicode/utf8" - "github.com/ewhal/nyaa/cache" "github.com/ewhal/nyaa/common" "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/model" @@ -118,69 +117,65 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( orderBy += "desc" } - tor, count, err = cache.Get(search, func() (tor []model.Torrent, count int, err error) { - parameters := serviceBase.WhereParams{ - Params: make([]interface{}, 0, 64), - } - conditions := make([]string, 0, 64) - if search.Category.Main != 0 { - conditions = append(conditions, "category = ?") - parameters.Params = append(parameters.Params, string(catString[0])) - } - if search.UserID != 0 { - conditions = append(conditions, "uploader = ?") - parameters.Params = append(parameters.Params, search.UserID) - } - if search.Category.Sub != 0 { - conditions = append(conditions, "sub_category = ?") - parameters.Params = append(parameters.Params, string(catString[2])) - } - if search.Status != 0 { - if search.Status == common.FilterRemakes { - conditions = append(conditions, "status > ?") - } else { - conditions = append(conditions, "status >= ?") - } - parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1)) - } - if len(search.NotNull) > 0 { - conditions = append(conditions, search.NotNull) - } - searchQuerySplit := strings.Fields(search.Query) - for i, word := range searchQuerySplit { - firstRune, _ := utf8.DecodeRuneInString(word) - if len(word) == 1 && unicode.IsPunct(firstRune) { - // some queries have a single punctuation character - // which causes a full scan instead of using the index - // and yields no meaningful results. - // due to len() == 1 we're just looking at 1-byte/ascii - // punctuation characters. - continue - } - - // SQLite has case-insensitive LIKE, but no ILIKE - var operator string - if db.ORM.Dialect().GetName() == "sqlite3" { - operator = "LIKE ?" - } else { - operator = "ILIKE ?" - } - - // TODO: make this faster ? - conditions = append(conditions, "torrent_name "+operator) - parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") - } - - parameters.Conditions = strings.Join(conditions[:], " AND ") - log.Infof("SQL query is :: %s\n", parameters.Conditions) - if countAll { - tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) + parameters := serviceBase.WhereParams{ + Params: make([]interface{}, 0, 64), + } + conditions := make([]string, 0, 64) + if search.Category.Main != 0 { + conditions = append(conditions, "category = ?") + parameters.Params = append(parameters.Params, string(catString[0])) + } + if search.UserID != 0 { + conditions = append(conditions, "uploader = ?") + parameters.Params = append(parameters.Params, search.UserID) + } + if search.Category.Sub != 0 { + conditions = append(conditions, "sub_category = ?") + parameters.Params = append(parameters.Params, string(catString[2])) + } + if search.Status != 0 { + if search.Status == common.FilterRemakes { + conditions = append(conditions, "status > ?") } else { - tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) + conditions = append(conditions, "status >= ?") + } + parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1)) + } + if len(search.NotNull) > 0 { + conditions = append(conditions, search.NotNull) + } + searchQuerySplit := strings.Fields(search.Query) + for i, word := range searchQuerySplit { + firstRune, _ := utf8.DecodeRuneInString(word) + if len(word) == 1 && unicode.IsPunct(firstRune) { + // some queries have a single punctuation character + // which causes a full scan instead of using the index + // and yields no meaningful results. + // due to len() == 1 we're just looking at 1-byte/ascii + // punctuation characters. + continue } - return - }) + // SQLite has case-insensitive LIKE, but no ILIKE + var operator string + if db.ORM.Dialect().GetName() == "sqlite3" { + operator = "LIKE ?" + } else { + operator = "ILIKE ?" + } + + // TODO: make this faster ? + conditions = append(conditions, "torrent_name "+operator) + parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") + } + + parameters.Conditions = strings.Join(conditions[:], " AND ") + log.Infof("SQL query is :: %s\n", parameters.Conditions) + if countAll { + tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) + } else { + tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) + } return } From 059ea7d2a8a80c60ceb9f7f3cad3391df2e73b95 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 11 May 2017 09:01:53 -0400 Subject: [PATCH 06/27] abstract out cache --- cache/cache.go | 139 ++++++----------------------------- cache/memcache/memcache.go | 21 ++++++ cache/native/native.go | 143 +++++++++++++++++++++++++++++++++++++ cache/nop/nop.go | 22 ++++++ config/cache.go | 14 ++++ config/config.go | 5 +- main.go | 6 +- router/homeHandler.go | 2 +- router/upload.go | 4 +- service/torrent/torrent.go | 2 + util/search/search.go | 2 +- 11 files changed, 237 insertions(+), 123 deletions(-) create mode 100644 cache/memcache/memcache.go create mode 100644 cache/native/native.go create mode 100644 cache/nop/nop.go create mode 100644 config/cache.go diff --git a/cache/cache.go b/cache/cache.go index b07f2d40..53c23690 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -1,132 +1,37 @@ package cache import ( - "container/list" - "sync" - "time" - + "github.com/ewhal/nyaa/cache/memcache" + "github.com/ewhal/nyaa/cache/native" + "github.com/ewhal/nyaa/cache/nop" "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/model" + + "errors" ) -const expiryTime = time.Minute - -var ( - cache = make(map[common.SearchParam]*list.Element, 10) - ll = list.New() - totalUsed int - mu sync.Mutex - - // Size sets the maximum size of the cache before evicting unread data in MB - Size float64 = 1 << 10 -) - -// Key stores the ID of either a thread or board page -type Key struct { - LastN uint8 - Board string - ID uint64 +// Cache defines interface for caching search results +type Cache interface { + Get(key common.SearchParam, r func() ([]model.Torrent, int, error)) ([]model.Torrent, int, error) + ClearAll() } -// Single cache entry -type store struct { - sync.Mutex // Controls general access to the contents of the struct - lastFetched time.Time - key common.SearchParam - data []model.Torrent - count, size int -} +var ErrInvalidCacheDialect = errors.New("invalid cache dialect") -// Check the cache for and existing record. If miss, run fn to retrieve fresh -// values. -func Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) ( - data []model.Torrent, count int, err error, -) { - s := getStore(key) +// Impl cache implementation instance +var Impl Cache - // Also keeps multiple requesters from simultaneously requesting the same - // data - s.Lock() - defer s.Unlock() - - if s.isFresh() { - return s.data, s.count, nil - } - - data, count, err = fn() - if err != nil { +func Init(conf *config.CacheConfig) (err error) { + switch conf.Dialect { + case "native": + Impl = native.New(conf.Size) return + case "memcache": + Impl = memcache.New() + return + default: + Impl = nop.New() } - s.update(data, count) return } - -// Retrieve a store from the cache or create a new one -func getStore(k common.SearchParam) (s *store) { - mu.Lock() - defer mu.Unlock() - - el := cache[k] - if el == nil { - s = &store{key: k} - cache[k] = ll.PushFront(s) - } else { - ll.MoveToFront(el) - s = el.Value.(*store) - } - return s -} - -// Clear the cache. Only used for testing. -func Clear() { - mu.Lock() - defer mu.Unlock() - - ll = list.New() - cache = make(map[common.SearchParam]*list.Element, 10) -} - -// Update the total used memory counter and evict, if over limit -func updateUsedSize(delta int) { - mu.Lock() - defer mu.Unlock() - - totalUsed += delta - - for totalUsed > int(Size)<<20 { - e := ll.Back() - if e == nil { - break - } - s := ll.Remove(e).(*store) - delete(cache, s.key) - totalUsed -= s.size - } -} - -// Return, if the data can still be considered fresh, without querying the DB -func (s *store) isFresh() bool { - if s.lastFetched.IsZero() { // New store - return false - } - return s.lastFetched.Add(expiryTime).After(time.Now()) -} - -// Stores the new values of s. Calculates and stores the new size. Passes the -// delta to the central cache to fire eviction checks. -func (s *store) update(data []model.Torrent, count int) { - newSize := 0 - for _, d := range data { - newSize += d.Size() - } - s.data = data - s.count = count - delta := newSize - s.size - s.size = newSize - s.lastFetched = time.Now() - - // Technically it is possible to update the size even when the store is - // already evicted, but that should never happen, unless you have a very - // small cache, very large stored datasets and a lot of traffic. - updateUsedSize(delta) -} diff --git a/cache/memcache/memcache.go b/cache/memcache/memcache.go new file mode 100644 index 00000000..57f36d96 --- /dev/null +++ b/cache/memcache/memcache.go @@ -0,0 +1,21 @@ +package memcache + +import ( + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/model" +) + +type Memcache struct { +} + +func (c *Memcache) Get(key common.SearchParam, r func() ([]model.Torrent, int, error)) (torrents []model.Torrent, num int, err error) { + return +} + +func (c *Memcache) ClearAll() { + +} + +func New() *Memcache { + return &Memcache{} +} diff --git a/cache/native/native.go b/cache/native/native.go new file mode 100644 index 00000000..c9b3ba36 --- /dev/null +++ b/cache/native/native.go @@ -0,0 +1,143 @@ +package native + +import ( + "container/list" + "sync" + "time" + + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/model" +) + +const expiryTime = time.Minute + +// NativeCache implements cache.Cache +type NativeCache struct { + cache map[common.SearchParam]*list.Element + ll *list.List + totalUsed int + mu sync.Mutex + + // Size sets the maximum size of the cache before evicting unread data in MB + Size float64 +} + +// New Creates New Native Cache instance +func New(sz float64) *NativeCache { + return &NativeCache{ + cache: make(map[common.SearchParam]*list.Element, 10), + Size: sz, + ll: list.New(), + } +} + +// Key stores the ID of either a thread or board page +type Key struct { + LastN uint8 + Board string + ID uint64 +} + +// Single cache entry +type store struct { + sync.Mutex // Controls general access to the contents of the struct + lastFetched time.Time + key common.SearchParam + data []model.Torrent + count, size int + n *NativeCache +} + +// Check the cache for and existing record. If miss, run fn to retrieve fresh +// values. +func (n *NativeCache) Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) ( + data []model.Torrent, count int, err error, +) { + s := n.getStore(key) + + // Also keeps multiple requesters from simultaneously requesting the same + // data + s.Lock() + defer s.Unlock() + + if s.isFresh() { + return s.data, s.count, nil + } + + data, count, err = fn() + if err != nil { + return + } + s.update(data, count) + return +} + +// Retrieve a store from the cache or create a new one +func (n *NativeCache) getStore(k common.SearchParam) (s *store) { + n.mu.Lock() + defer n.mu.Unlock() + + el := n.cache[k] + if el == nil { + s = &store{key: k, n: n} + n.cache[k] = n.ll.PushFront(s) + } else { + n.ll.MoveToFront(el) + s = el.Value.(*store) + } + return s +} + +// Clear the cache. Only used for testing. +func (n *NativeCache) ClearAll() { + n.mu.Lock() + defer n.mu.Unlock() + + n.ll = list.New() + n.cache = make(map[common.SearchParam]*list.Element, 10) +} + +// Update the total used memory counter and evict, if over limit +func (n *NativeCache) updateUsedSize(delta int) { + n.mu.Lock() + defer n.mu.Unlock() + + n.totalUsed += delta + + for n.totalUsed > int(n.Size)<<20 { + e := n.ll.Back() + if e == nil { + break + } + s := n.ll.Remove(e).(*store) + delete(n.cache, s.key) + n.totalUsed -= s.size + } +} + +// Return, if the data can still be considered fresh, without querying the DB +func (s *store) isFresh() bool { + if s.lastFetched.IsZero() { // New store + return false + } + return s.lastFetched.Add(expiryTime).After(time.Now()) +} + +// Stores the new values of s. Calculates and stores the new size. Passes the +// delta to the central cache to fire eviction checks. +func (s *store) update(data []model.Torrent, count int) { + newSize := 0 + for _, d := range data { + newSize += d.Size() + } + s.data = data + s.count = count + delta := newSize - s.size + s.size = newSize + s.lastFetched = time.Now() + + // Technically it is possible to update the size even when the store is + // already evicted, but that should never happen, unless you have a very + // small cache, very large stored datasets and a lot of traffic. + s.n.updateUsedSize(delta) +} diff --git a/cache/nop/nop.go b/cache/nop/nop.go new file mode 100644 index 00000000..21c10580 --- /dev/null +++ b/cache/nop/nop.go @@ -0,0 +1,22 @@ +package nop + +import ( + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/model" +) + +type NopCache struct { +} + +func (c *NopCache) Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) ([]model.Torrent, int, error) { + return fn() +} + +func (c *NopCache) ClearAll() { + +} + +// New creates a new Cache that does NOTHING :D +func New() *NopCache { + return &NopCache{} +} diff --git a/config/cache.go b/config/cache.go new file mode 100644 index 00000000..93e195fb --- /dev/null +++ b/config/cache.go @@ -0,0 +1,14 @@ +package config + +// CacheConfig is config struct for caching strategy +type CacheConfig struct { + Dialect string + URL string + Size float64 +} + +const DefaultCacheSize = 1 << 10 + +var DefaultCacheConfig = CacheConfig{ + Dialect: "nop", +} diff --git a/config/config.go b/config/config.go index bea5b8d9..1b828be1 100644 --- a/config/config.go +++ b/config/config.go @@ -24,11 +24,13 @@ type Config struct { DBParams string `json:"db_params"` // tracker scraper config (required) Scrape ScraperConfig `json:"scraper"` + // cache config + Cache CacheConfig `json:"cache"` // optional i2p configuration I2P *I2PConfig `json:"i2p"` } -var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", DefaultScraperConfig, nil} +var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", DefaultScraperConfig, DefaultCacheConfig, nil} var allowedDatabaseTypes = map[string]bool{ "sqlite3": true, @@ -44,6 +46,7 @@ func New() *Config { config.DBType = Defaults.DBType config.DBParams = Defaults.DBParams config.Scrape = Defaults.Scrape + config.Cache = Defaults.Cache return &config } diff --git a/main.go b/main.go index 8c92a00c..c9332303 100644 --- a/main.go +++ b/main.go @@ -98,7 +98,7 @@ func main() { processFlags := conf.BindFlags() defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout") mode := flag.String("mode", "webapp", "which mode to run daemon in, either webapp or scraper") - flag.Float64Var(&cache.Size, "c", cache.Size, "size of the search cache in MB") + flag.Float64Var(&conf.Cache.Size, "c", config.DefaultCacheSize, "size of the search cache in MB") flag.Parse() if *defaults { stdout := bufio.NewWriter(os.Stdout) @@ -121,6 +121,10 @@ func main() { log.Fatal(err.Error()) } initI18N() + err = cache.Init(&conf.Cache) + if err != nil { + log.Fatal(err.Error()) + } go signals.Handle() if len(config.TorrentFileStorage) > 0 { err := os.MkdirAll(config.TorrentFileStorage, 0700) diff --git a/router/homeHandler.go b/router/homeHandler.go index ef2156e0..a06b5209 100644 --- a/router/homeHandler.go +++ b/router/homeHandler.go @@ -43,7 +43,7 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) { Max: uint(maxPerPage), Page: pagenum, } - torrents, nbTorrents, err := cache.Get(search, func() ([]model.Torrent, int, error) { + torrents, nbTorrents, err := cache.Impl.Get(search, func() ([]model.Torrent, int, error) { torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) if !log.CheckError(err) { util.SendError(w, err, 400) diff --git a/router/upload.go b/router/upload.go index 1eeb3b5c..42664718 100644 --- a/router/upload.go +++ b/router/upload.go @@ -30,7 +30,7 @@ type UploadForm struct { Category string Remake bool Description string - Status int + Status int captcha.Captcha Infohash string @@ -93,7 +93,7 @@ func (f *UploadForm) ExtractInfo(r *http.Request) error { f.Name = util.TrimWhitespaces(f.Name) f.Description = p.Sanitize(util.TrimWhitespaces(f.Description)) f.Magnet = util.TrimWhitespaces(f.Magnet) - cache.Clear() + cache.Impl.ClearAll() catsSplit := strings.Split(f.Category, "_") // need this to prevent out of index panics diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go index caa776f7..bef626d7 100644 --- a/service/torrent/torrent.go +++ b/service/torrent/torrent.go @@ -11,6 +11,7 @@ import ( "github.com/ewhal/nyaa/model" "github.com/ewhal/nyaa/service" "github.com/ewhal/nyaa/util" + "github.com/ewhal/nyaa/util/log" ) /* Function to interact with Models @@ -140,6 +141,7 @@ func getTorrentsOrderBy(parameters *serviceBase.WhereParams, orderBy string, lim if limit != 0 || offset != 0 { // if limits provided dbQuery = dbQuery + " LIMIT " + strconv.Itoa(limit) + " OFFSET " + strconv.Itoa(offset) } + log.Infof("SQL: %s", dbQuery) err = db.ORM.Raw(dbQuery, params...).Find(&torrents).Error return } diff --git a/util/search/search.go b/util/search/search.go index ba66540f..33388583 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -118,7 +118,7 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( orderBy += "desc" } - tor, count, err = cache.Get(search, func() (tor []model.Torrent, count int, err error) { + tor, count, err = cache.Impl.Get(search, func() (tor []model.Torrent, count int, err error) { parameters := serviceBase.WhereParams{ Params: make([]interface{}, 0, 64), } From 521064b946b15e1f9d7d33b93cad1246af77b860 Mon Sep 17 00:00:00 2001 From: Eliot Whalan Date: Thu, 11 May 2017 23:12:19 +1000 Subject: [PATCH 07/27] Add vary accept-language header --- util/languages/translation.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/util/languages/translation.go b/util/languages/translation.go index 7b45a7e9..2be97bfb 100644 --- a/util/languages/translation.go +++ b/util/languages/translation.go @@ -2,8 +2,8 @@ package languages import ( "fmt" - "github.com/nicksnyder/go-i18n/i18n" "github.com/ewhal/nyaa/service/user" + "github.com/nicksnyder/go-i18n/i18n" "html/template" "net/http" ) @@ -22,7 +22,7 @@ func TfuncWithFallback(language string, languages ...string) (i18n.TranslateFunc if err1 != nil && err2 != nil { // fallbackT is still a valid function even with the error, it returns translationID. - return fallbackT, err2; + return fallbackT, err2 } return func(translationID string, args ...interface{}) string { @@ -42,7 +42,7 @@ func GetAvailableLanguages() (languages map[string]string) { /* Translation files should have an ID with the translated language name. If they don't, just use the languageTag */ if languageName := T("language_name"); languageName != "language_name" { - languages[languageTag] = languageName; + languages[languageTag] = languageName } else { languages[languageTag] = languageTag } @@ -67,7 +67,7 @@ func SetTranslationFromRequest(tmpl *template.Template, r *http.Request, default userLanguage := "" user, _, err := userService.RetrieveCurrentUser(r) if err == nil { - userLanguage = user.Language; + userLanguage = user.Language } cookie, err := r.Cookie("lang") @@ -78,5 +78,6 @@ func SetTranslationFromRequest(tmpl *template.Template, r *http.Request, default // go-i18n supports the format of the Accept-Language header, thankfully. headerLanguage := r.Header.Get("Accept-Language") + r.Header.Add("Vary", "Accept-Encoding") return SetTranslation(tmpl, userLanguage, cookieLanguage, headerLanguage, defaultLanguage) } From 8e2c643a3478eea164fabdc3ea5066940483999f Mon Sep 17 00:00:00 2001 From: sfan5 Date: Thu, 11 May 2017 15:14:48 +0200 Subject: [PATCH 08/27] Fix missing error enforcement on comments and reports --- router/viewTorrentHandler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/router/viewTorrentHandler.go b/router/viewTorrentHandler.go index 0ac9f8f3..80cf91ea 100644 --- a/router/viewTorrentHandler.go +++ b/router/viewTorrentHandler.go @@ -41,11 +41,13 @@ func PostCommentHandler(w http.ResponseWriter, r *http.Request) { if strings.TrimSpace(r.FormValue("comment")) == "" { http.Error(w, "comment empty", 406) + return } userCaptcha := captcha.Extract(r) if !captcha.Authenticate(userCaptcha) { http.Error(w, "bad captcha", 403) + return } currentUser := GetUser(r) content := p.Sanitize(r.FormValue("comment")) @@ -76,6 +78,7 @@ func ReportTorrentHandler(w http.ResponseWriter, r *http.Request) { userCaptcha := captcha.Extract(r) if !captcha.Authenticate(userCaptcha) { http.Error(w, "bad captcha", 403) + return } currentUser := GetUser(r) From 95562ecd60f2790048315461846a2fa61965aaba Mon Sep 17 00:00:00 2001 From: sfan5 Date: Thu, 11 May 2017 15:20:19 +0200 Subject: [PATCH 09/27] Fix S/L/C translations on view page --- translations/de-de.all.json | 12 ++++++++++++ translations/en-us.all.json | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/translations/de-de.all.json b/translations/de-de.all.json index 84c063ad..a7dd3c35 100644 --- a/translations/de-de.all.json +++ b/translations/de-de.all.json @@ -590,5 +590,17 @@ { "id": "torrent_status_remake", "translation": "Remake" + }, + { + "id": "seeders", + "translation": "Seeder" + }, + { + "id": "leechers", + "translation": "Leecher" + }, + { + "id": "completed", + "translation": "Komplett" } ] diff --git a/translations/en-us.all.json b/translations/en-us.all.json index 6c94d757..bbf97799 100644 --- a/translations/en-us.all.json +++ b/translations/en-us.all.json @@ -606,5 +606,17 @@ { "id":"date_format", "translation": "2006-01-02 15:04" + }, + { + "id": "seeders", + "translation": "Seeders" + }, + { + "id": "leechers", + "translation": "Leechers" + }, + { + "id": "completed", + "translation": "Completed" } ] From 2057a4aa5134f0787d3f812c650e0afa39b43bc5 Mon Sep 17 00:00:00 2001 From: sisimouto Date: Thu, 11 May 2017 22:28:29 +0900 Subject: [PATCH 10/27] Update th-th.all.json --- translations/th-th.all.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/translations/th-th.all.json b/translations/th-th.all.json index a932b009..cc5e6857 100644 --- a/translations/th-th.all.json +++ b/translations/th-th.all.json @@ -602,5 +602,21 @@ { "id": "profile_edit_page", "translation": "แก้ไขโปรไฟล์ของ %s" + }, + { + "id":"date_format", + "translation": "2006/01/02 15:04" + }, + { + "id": "seeders", + "translation": "ผู้ปล่อย" + }, + { + "id": "leechers", + "translation": "ผู้โหลด" + }, + { + "id": "completed", + "translation": "โหลดเสร็จแล้ว" } ] From 8b6ddd95c20bf5640951377880604116ca078d40 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 11 May 2017 09:29:08 -0400 Subject: [PATCH 11/27] forgot file --- config/search.go | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 config/search.go diff --git a/config/search.go b/config/search.go new file mode 100644 index 00000000..19eabbdc --- /dev/null +++ b/config/search.go @@ -0,0 +1,6 @@ +package config + +type SearchConfig struct { +} + +var DefaultSearchConfig = SearchConfig{} From 217038eea008f8e3d4af540a7843f2e03ac5730e Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 11 May 2017 09:40:33 -0400 Subject: [PATCH 12/27] only scrape torrents uploaded within 90 days --- service/scraper/scraper.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/service/scraper/scraper.go b/service/scraper/scraper.go index 7d4b8e61..61a5825e 100644 --- a/service/scraper/scraper.go +++ b/service/scraper/scraper.go @@ -179,8 +179,9 @@ func (sc *Scraper) removeStale() { func (sc *Scraper) Scrape(packets uint) { now := time.Now().Add(0 - sc.interval) - - rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM torrents WHERE last_scrape IS NULL OR last_scrape < ? ORDER BY torrent_id DESC LIMIT ?", now, packets*ScrapesPerPacket).Rows() + // only scrape torretns uploaded within 90 days + oldest := now.Add(0 - (time.Hour * 24 * 90)) + rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM torrents WHERE last_scrape IS NULL OR last_scrape < ? AND date > ? ORDER BY torrent_id DESC LIMIT ?", now, oldest, packets*ScrapesPerPacket).Rows() if err == nil { counter := 0 var scrape [ScrapesPerPacket]model.Torrent From 4d5c04bbac8a5e1325624e5a7c181fb064234ab9 Mon Sep 17 00:00:00 2001 From: akuma06 Date: Thu, 11 May 2017 15:50:26 +0200 Subject: [PATCH 13/27] Added CROS for RSS --- router/rssHandler.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/router/rssHandler.go b/router/rssHandler.go index 422b0622..c88a16fd 100644 --- a/router/rssHandler.go +++ b/router/rssHandler.go @@ -36,12 +36,13 @@ func RSSHandler(w http.ResponseWriter, r *http.Request) { Id: "https://" + config.WebAddress + "/view/" + strconv.FormatUint(uint64(torrents[i].ID), 10), Title: torrents[i].Name, Link: &feeds.Link{Href: string(torrentJSON.Magnet)}, - Description: "", + Description: string(torrentJSON.Description), Created: torrents[0].Date, Updated: torrents[0].Date, } } - + //allow cross domain AJAX requests + w.Header().Set("Access-Control-Allow-Origin", "*") rss, rssErr := feed.ToRss() if rssErr != nil { http.Error(w, err.Error(), http.StatusInternalServerError) From 63bfc8dc0482f31bd28eb2a88d23c03dcc02878f Mon Sep 17 00:00:00 2001 From: PantsuDev Date: Thu, 11 May 2017 23:50:46 +1000 Subject: [PATCH 14/27] Update README.md --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ced9ed42..4db77e20 100644 --- a/README.md +++ b/README.md @@ -64,17 +64,18 @@ Access the website by going to [localhost:9999](http://localhost:9999). > nyaa_psql.backup. ## TODO - * improve scraping - * fix up cache bug - * import sukebei torrents into db/work on sukebei + * sukebei + * get sukebei_torrents table working + * add config option for sukebei or maybe make things all in one + * sukebei categories and category images + * Get code up to standard of go lint recommendations * Write tests - * fix sukebei categories - * Daily DB dumps - * Site theme + + * Site theme * original nyaa theme * API improvement - * Scraping of fan subbing RSS feeds + * Scraping of fan subbing RSS feeds(WIP) # LICENSE This project is licensed under the MIT License - see the LICENSE.md file for details From d053b18f34240de98e8ed373da2072d2b407818d Mon Sep 17 00:00:00 2001 From: Myrmece Date: Thu, 11 May 2017 15:52:19 +0200 Subject: [PATCH 15/27] Updated translation. --- translations/fr-fr.all.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/translations/fr-fr.all.json b/translations/fr-fr.all.json index 5bdc20d1..a9709341 100644 --- a/translations/fr-fr.all.json +++ b/translations/fr-fr.all.json @@ -606,5 +606,21 @@ { "id": "profile_edit_page", "translation": "Éditer le profil de %s" + }, + { + "id":"date_format", + "translation": "2006-01-02 15:04" + }, + { + "id": "seeders", + "translation": "Seeders" + }, + { + "id": "leechers", + "translation": "Leechers" + }, + { + "id": "completed", + "translation": "Terminé" } ] From c8d94c6e5bc2e69ad31bb3aa5d87e63b05f6c3d7 Mon Sep 17 00:00:00 2001 From: Nutjob Date: Thu, 11 May 2017 16:13:56 +0200 Subject: [PATCH 16/27] Update it-it.all.json --- translations/it-it.all.json | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/translations/it-it.all.json b/translations/it-it.all.json index 40232b8f..fecd5068 100644 --- a/translations/it-it.all.json +++ b/translations/it-it.all.json @@ -602,5 +602,21 @@ { "id": "profile_edit_page", "translation": "Modifica il profilo di %s" - } + }, + { + "id":"date_format", + "translation": "2006-01-02 15:04" + }, + { + "id": "seeders", + "translation": "Seeders" + }, + { + "id": "leechers", + "translation": "Leechers" + }, + { + "id": "completed", + "translation": "Completato" +} ] From 100ecffda7eddd96d2b705d837e187d347c71976 Mon Sep 17 00:00:00 2001 From: Jeff Date: Thu, 11 May 2017 15:06:47 -0400 Subject: [PATCH 17/27] fix scraper, optimize updates so it doesn't suck massive ass (#367) * fix scraper, optimize updates so it doesn't suck massive ass * fucking ass --- db/gorm.go | 4 ++++ service/scraper/bucket.go | 2 +- service/scraper/scraper.go | 16 ++++++++++++---- service/scraper/transaction.go | 22 +++++++++++++--------- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/db/gorm.go b/db/gorm.go index 30aa729e..77fa33a6 100644 --- a/db/gorm.go +++ b/db/gorm.go @@ -11,6 +11,8 @@ import ( var ORM *gorm.DB +var IsSqlite bool + // GormInit init gorm ORM. func GormInit(conf *config.Config) (*gorm.DB, error) { db, openErr := gorm.Open(conf.DBType, conf.DBParams) @@ -19,6 +21,8 @@ func GormInit(conf *config.Config) (*gorm.DB, error) { return nil, openErr } + IsSqlite = conf.DBType == "sqlite" + connectionErr := db.DB().Ping() if connectionErr != nil { log.CheckError(connectionErr) diff --git a/service/scraper/bucket.go b/service/scraper/bucket.go index d9c85372..bf8c34f9 100644 --- a/service/scraper/bucket.go +++ b/service/scraper/bucket.go @@ -31,7 +31,7 @@ func (b *Bucket) NewTransaction(swarms []model.Torrent) (t *Transaction) { swarms: make([]model.Torrent, len(swarms)), state: stateSendID, } - copy(t.swarms, swarms) + copy(t.swarms[:], swarms[:]) b.transactions[id] = t b.access.Unlock() return diff --git a/service/scraper/scraper.go b/service/scraper/scraper.go index 61a5825e..28797dd8 100644 --- a/service/scraper/scraper.go +++ b/service/scraper/scraper.go @@ -36,9 +36,9 @@ func New(conf *config.ScraperConfig) (sc *Scraper, err error) { recvQueue: make(chan *RecvEvent, 1024), errQueue: make(chan error), trackers: make(map[string]*Bucket), - ticker: time.NewTicker(time.Second), + ticker: time.NewTicker(time.Second * 10), interval: time.Second * time.Duration(conf.IntervalSeconds), - cleanup: time.NewTicker(time.Second), + cleanup: time.NewTicker(time.Minute), } if sc.PacketsPerSecond == 0 { @@ -181,7 +181,7 @@ func (sc *Scraper) Scrape(packets uint) { now := time.Now().Add(0 - sc.interval) // only scrape torretns uploaded within 90 days oldest := now.Add(0 - (time.Hour * 24 * 90)) - rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM torrents WHERE last_scrape IS NULL OR last_scrape < ? AND date > ? ORDER BY torrent_id DESC LIMIT ?", now, oldest, packets*ScrapesPerPacket).Rows() + rows, err := db.ORM.Raw("SELECT torrent_id, torrent_hash FROM torrents WHERE ( last_scrape IS NULL OR last_scrape < ? ) AND date > ? ORDER BY torrent_id DESC LIMIT ?", now, oldest, packets*ScrapesPerPacket).Rows() if err == nil { counter := 0 var scrape [ScrapesPerPacket]model.Torrent @@ -189,13 +189,21 @@ func (sc *Scraper) Scrape(packets uint) { idx := counter % ScrapesPerPacket rows.Scan(&scrape[idx].ID, &scrape[idx].Hash) counter++ - if idx == 0 { + if counter%ScrapesPerPacket == 0 { for _, b := range sc.trackers { t := b.NewTransaction(scrape[:]) sc.sendQueue <- t.SendEvent(b.Addr) } } } + idx := counter % ScrapesPerPacket + if idx > 0 { + for _, b := range sc.trackers { + t := b.NewTransaction(scrape[:idx]) + sc.sendQueue <- t.SendEvent(b.Addr) + } + } + log.Infof("scrape %d", counter) rows.Close() } else { diff --git a/service/scraper/transaction.go b/service/scraper/transaction.go index 7b7de953..aaf6ece3 100644 --- a/service/scraper/transaction.go +++ b/service/scraper/transaction.go @@ -53,18 +53,22 @@ func (t *Transaction) handleScrapeReply(data []byte) { } } +const pgQuery = "UPDATE torrents SET seeders = $1 , leechers = $2 , completed = $3 , last_scrape = $4 WHERE torrent_id = $5" +const sqliteQuery = "UPDATE torrents SET seeders = ? , leechers = ? , completed = ? , last_scrape = ? WHERE torrent_id = ?" + // Sync syncs models with database func (t *Transaction) Sync() (err error) { - for idx := range t.swarms { - err = db.ORM.Model(&t.swarms[idx]).Updates(map[string]interface{}{ - "seeders": t.swarms[idx].Seeders, - "leechers": t.swarms[idx].Leechers, - "completed": t.swarms[idx].Completed, - "last_scrape": t.swarms[idx].LastScrape, - }).Error - if err != nil { - break + q := pgQuery + if db.IsSqlite { + q = sqliteQuery + } + tx, e := db.ORM.DB().Begin() + err = e + if err == nil { + for idx := range t.swarms { + _, err = tx.Exec(q, t.swarms[idx].Seeders, t.swarms[idx].Leechers, t.swarms[idx].Completed, t.swarms[idx].LastScrape, t.swarms[idx].ID) } + tx.Commit() } return } From a377e272c8eb98bdec665308a4e3075e3f290732 Mon Sep 17 00:00:00 2001 From: bittebitte Date: Thu, 11 May 2017 14:25:16 -0500 Subject: [PATCH 18/27] Autofocus username field --- templates/user/login.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/user/login.html b/templates/user/login.html index 7ac16973..09e09c4f 100644 --- a/templates/user/login.html +++ b/templates/user/login.html @@ -12,7 +12,7 @@
{{ . }}
{{end}}
- + {{ range (index $.FormErrors "username")}}

{{ . }}

{{end}} From bfb9bf32390abaeeb3413076015ed3acda8e2ca0 Mon Sep 17 00:00:00 2001 From: tomleb Date: Thu, 11 May 2017 15:46:14 -0400 Subject: [PATCH 19/27] Fix empty comment (sanitize then trim and check) (#368) --- router/viewTorrentHandler.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/router/viewTorrentHandler.go b/router/viewTorrentHandler.go index 80cf91ea..7e467a50 100644 --- a/router/viewTorrentHandler.go +++ b/router/viewTorrentHandler.go @@ -39,11 +39,6 @@ func PostCommentHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id := vars["id"] - if strings.TrimSpace(r.FormValue("comment")) == "" { - http.Error(w, "comment empty", 406) - return - } - userCaptcha := captcha.Extract(r) if !captcha.Authenticate(userCaptcha) { http.Error(w, "bad captcha", 403) @@ -52,6 +47,11 @@ func PostCommentHandler(w http.ResponseWriter, r *http.Request) { currentUser := GetUser(r) content := p.Sanitize(r.FormValue("comment")) + if strings.TrimSpace(content) == "" { + http.Error(w, "comment empty", 406) + return + } + idNum, err := strconv.Atoi(id) userID := currentUser.ID From 4b810494f3b52f086574a5023404fd6867854867 Mon Sep 17 00:00:00 2001 From: wranai Date: Thu, 11 May 2017 21:46:23 +0200 Subject: [PATCH 20/27] Now really fixing the status filter (#354) I managed to screw up in the last update. Normal is 0 and remake is 1, so that makes the "goodness" order of statuses non-monotonic, so I can't use the greater sign for filtering out remakes... --- util/search/search.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/search/search.go b/util/search/search.go index 60472a8f..1b96529b 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -149,7 +149,7 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( } if search.Status != 0 { if search.Status == common.FilterRemakes { - conditions = append(conditions, "status > ?") + conditions = append(conditions, "status <> ?") } else { conditions = append(conditions, "status >= ?") } @@ -194,4 +194,4 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( return }) return -} +} \ No newline at end of file From 61ba31b337dc837426d508626059b4661dd58acb Mon Sep 17 00:00:00 2001 From: akuma06 Date: Thu, 11 May 2017 22:21:06 +0200 Subject: [PATCH 21/27] No more gzip --- router/router.go | 148 +++++++++++++++++++++++------------------------ 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/router/router.go b/router/router.go index ffdc26ef..8f93b335 100755 --- a/router/router.go +++ b/router/router.go @@ -4,7 +4,7 @@ import ( "net/http" "github.com/ewhal/nyaa/service/captcha" - "github.com/gorilla/handlers" + // "github.com/gorilla/handlers" "github.com/gorilla/mux" ) @@ -15,102 +15,102 @@ func init() { cssHandler := http.FileServer(http.Dir("./public/css/")) jsHandler := http.FileServer(http.Dir("./public/js/")) imgHandler := http.FileServer(http.Dir("./public/img/")) - + gzipHomeHandler := http.HandlerFunc(HomeHandler) + gzipAPIHandler := http.HandlerFunc(ApiHandler) + gzipAPIViewHandler := http.HandlerFunc(ApiViewHandler) + gzipViewHandler := http.HandlerFunc(ViewHandler) + gzipUserProfileHandler := http.HandlerFunc(UserProfileHandler) + gzipUserDetailsHandler := http.HandlerFunc(UserDetailsHandler) + gzipUserProfileFormHandler := http.HandlerFunc(UserProfileFormHandler) +/* // Enable GZIP compression for all handlers except imgHandler and captcha - gzipCSSHandler := handlers.CompressHandler(cssHandler) - gzipJSHandler := handlers.CompressHandler(jsHandler) - gzipHomeHandler := handlers.CompressHandler(http.HandlerFunc(HomeHandler)) - gzipSearchHandler := handlers.CompressHandler(http.HandlerFunc(SearchHandler)) - gzipAPIHandler := handlers.CompressHandler(http.HandlerFunc(ApiHandler)) - gzipAPIViewHandler := handlers.CompressHandler(http.HandlerFunc(ApiViewHandler)) - gzipAPIUploadHandler := handlers.CompressHandler(http.HandlerFunc(ApiUploadHandler)) - gzipAPIUpdateHandler := handlers.CompressHandler(http.HandlerFunc(ApiUpdateHandler)) - gzipFaqHandler := handlers.CompressHandler(http.HandlerFunc(FaqHandler)) - gzipRSSHandler := handlers.CompressHandler(http.HandlerFunc(RSSHandler)) - gzipViewHandler := handlers.CompressHandler(http.HandlerFunc(ViewHandler)) - gzipUploadHandler := handlers.CompressHandler(http.HandlerFunc(UploadHandler)) - gzipUserRegisterFormHandler := handlers.CompressHandler(http.HandlerFunc(UserRegisterFormHandler)) - gzipUserLoginFormHandler := handlers.CompressHandler(http.HandlerFunc(UserLoginFormHandler)) - gzipUserVerifyEmailHandler := handlers.CompressHandler(http.HandlerFunc(UserVerifyEmailHandler)) - gzipUserRegisterPostHandler := handlers.CompressHandler(http.HandlerFunc(UserRegisterPostHandler)) - gzipUserLoginPostHandler := handlers.CompressHandler(http.HandlerFunc(UserLoginPostHandler)) - gzipUserLogoutHandler := handlers.CompressHandler(http.HandlerFunc(UserLogoutHandler)) - gzipUserProfileHandler := handlers.CompressHandler(http.HandlerFunc(UserProfileHandler)) - gzipUserFollowHandler := handlers.CompressHandler(http.HandlerFunc(UserFollowHandler)) - gzipUserDetailsHandler := handlers.CompressHandler(http.HandlerFunc(UserDetailsHandler)) - gzipUserProfileFormHandler := handlers.CompressHandler(http.HandlerFunc(UserProfileFormHandler)) + gzipCSSHandler := cssHandler) + gzipJSHandler:= jsHandler) + gzipSearchHandler:= http.HandlerFunc(SearchHandler) + gzipAPIUploadHandler := http.HandlerFunc(ApiUploadHandler) + gzipAPIUpdateHandler := http.HandlerFunc(ApiUpdateHandler) + gzipFaqHandler := http.HandlerFunc(FaqHandler) + gzipRSSHandler := http.HandlerFunc(RSSHandler) + gzipUploadHandler := http.HandlerFunc(UploadHandler) + gzipUserRegisterFormHandler := http.HandlerFunc(UserRegisterFormHandler) + gzipUserLoginFormHandler := http.HandlerFunc(UserLoginFormHandler) + gzipUserVerifyEmailHandler := http.HandlerFunc(UserVerifyEmailHandler) + gzipUserRegisterPostHandler := http.HandlerFunc(UserRegisterPostHandler) + gzipUserLoginPostHandler := http.HandlerFunc(UserLoginPostHandler) + gzipUserLogoutHandler := http.HandlerFunc(UserLogoutHandler) + gzipUserFollowHandler := http.HandlerFunc(UserFollowHandler) - gzipIndexModPanel := handlers.CompressHandler(http.HandlerFunc(IndexModPanel)) - gzipTorrentsListPanel := handlers.CompressHandler(http.HandlerFunc(TorrentsListPanel)) - gzipTorrentReportListPanel := handlers.CompressHandler(http.HandlerFunc(TorrentReportListPanel)) - gzipUsersListPanel := handlers.CompressHandler(http.HandlerFunc(UsersListPanel)) - gzipCommentsListPanel := handlers.CompressHandler(http.HandlerFunc(CommentsListPanel)) - gzipTorrentEditModPanel := handlers.CompressHandler(http.HandlerFunc(TorrentEditModPanel)) - gzipTorrentPostEditModPanel := handlers.CompressHandler(http.HandlerFunc(TorrentPostEditModPanel)) - gzipCommentDeleteModPanel := handlers.CompressHandler(http.HandlerFunc(CommentDeleteModPanel)) - gzipTorrentDeleteModPanel := handlers.CompressHandler(http.HandlerFunc(TorrentDeleteModPanel)) - gzipTorrentReportDeleteModPanel := handlers.CompressHandler(http.HandlerFunc(TorrentReportDeleteModPanel)) + gzipIndexModPanel := http.HandlerFunc(IndexModPanel) + gzipTorrentsListPanel := http.HandlerFunc(TorrentsListPanel) + gzipTorrentReportListPanel := http.HandlerFunc(TorrentReportListPanel) + gzipUsersListPanel := http.HandlerFunc(UsersListPanel) + gzipCommentsListPanel := http.HandlerFunc(CommentsListPanel) + gzipTorrentEditModPanel := http.HandlerFunc(TorrentEditModPanel) + gzipTorrentPostEditModPanel := http.HandlerFunc(TorrentPostEditModPanel) + gzipCommentDeleteModPanel := http.HandlerFunc(CommentDeleteModPanel) + gzipTorrentDeleteModPanel := http.HandlerFunc(TorrentDeleteModPanel) + gzipTorrentReportDeleteModPanel := http.HandlerFunc(TorrentReportDeleteModPanel)*/ - //gzipTorrentReportCreateHandler := handlers.CompressHandler(http.HandlerFunc(CreateTorrentReportHandler)) - //gzipTorrentReportDeleteHandler := handlers.CompressHandler(http.HandlerFunc(DeleteTorrentReportHandler)) - //gzipTorrentDeleteHandler := handlers.CompressHandler(http.HandlerFunc(DeleteTorrentHandler)) + //gzipTorrentReportCreateHandler := http.HandlerFunc(CreateTorrentReportHandler) + //gzipTorrentReportDeleteHandler := http.HandlerFunc(DeleteTorrentReportHandler) + //gzipTorrentDeleteHandler := http.HandlerFunc(DeleteTorrentHandler) Router = mux.NewRouter() // Routes - http.Handle("/css/", http.StripPrefix("/css/", wrapHandler(gzipCSSHandler))) - http.Handle("/js/", http.StripPrefix("/js/", wrapHandler(gzipJSHandler))) - http.Handle("/img/", http.StripPrefix("/img/", wrapHandler(imgHandler))) - Router.Handle("/", gzipHomeHandler).Name("home") + http.Handle("/css/", http.StripPrefix("/css/", cssHandler)) + http.Handle("/js/", http.StripPrefix("/js/", jsHandler)) + http.Handle("/img/", http.StripPrefix("/img/", imgHandler)) + Router.Handle("/", wrapHandler(gzipHomeHandler)).Name("home") Router.Handle("/page/{page:[0-9]+}", wrapHandler(gzipHomeHandler)).Name("home_page") - Router.Handle("/search", gzipSearchHandler).Name("search") - Router.Handle("/search/{page}", gzipSearchHandler).Name("search_page") - Router.Handle("/api", gzipAPIHandler).Methods("GET") + Router.HandleFunc("/search", SearchHandler).Name("search") + Router.HandleFunc("/search/{page}", SearchHandler).Name("search_page") + Router.Handle("/api", wrapHandler(gzipAPIHandler)).Methods("GET") Router.Handle("/api/{page:[0-9]*}", wrapHandler(gzipAPIHandler)).Methods("GET") Router.Handle("/api/view/{id}", wrapHandler(gzipAPIViewHandler)).Methods("GET") - Router.Handle("/api/upload", gzipAPIUploadHandler).Methods("POST") - Router.Handle("/api/update", gzipAPIUpdateHandler).Methods("PUT") - Router.Handle("/faq", gzipFaqHandler).Name("faq") - Router.Handle("/feed", gzipRSSHandler).Name("feed") + Router.HandleFunc("/api/upload", ApiUploadHandler).Methods("POST") + Router.HandleFunc("/api/update", ApiUpdateHandler).Methods("PUT") + Router.HandleFunc("/faq", FaqHandler).Name("faq") + Router.HandleFunc("/feed", RSSHandler).Name("feed") Router.Handle("/view/{id}", wrapHandler(gzipViewHandler)).Methods("GET").Name("view_torrent") Router.HandleFunc("/view/{id}", PostCommentHandler).Methods("POST").Name("post_comment") - Router.Handle("/upload", gzipUploadHandler).Name("upload") - Router.Handle("/user/register", gzipUserRegisterFormHandler).Name("user_register").Methods("GET") - Router.Handle("/user/login", gzipUserLoginFormHandler).Name("user_login").Methods("GET") - Router.Handle("/verify/email/{token}", gzipUserVerifyEmailHandler).Name("user_verify").Methods("GET") - Router.Handle("/user/register", gzipUserRegisterPostHandler).Name("user_register").Methods("POST") - Router.Handle("/user/login", gzipUserLoginPostHandler).Name("user_login").Methods("POST") - Router.Handle("/user/logout", gzipUserLogoutHandler).Name("user_logout") + Router.HandleFunc("/upload", UploadHandler).Name("upload") + Router.HandleFunc("/user/register", UserRegisterFormHandler).Name("user_register").Methods("GET") + Router.HandleFunc("/user/login", UserLoginFormHandler).Name("user_login").Methods("GET") + Router.HandleFunc("/verify/email/{token}", UserVerifyEmailHandler).Name("user_verify").Methods("GET") + Router.HandleFunc("/user/register", UserRegisterPostHandler).Name("user_register").Methods("POST") + Router.HandleFunc("/user/login", UserLoginPostHandler).Name("user_login").Methods("POST") + Router.HandleFunc("/user/logout", UserLogoutHandler).Name("user_logout") Router.Handle("/user/{id}/{username}", wrapHandler(gzipUserProfileHandler)).Name("user_profile").Methods("GET") - Router.Handle("/user/{id}/{username}/follow", gzipUserFollowHandler).Name("user_follow").Methods("GET") + Router.HandleFunc("/user/{id}/{username}/follow", UserFollowHandler).Name("user_follow").Methods("GET") Router.Handle("/user/{id}/{username}/edit", wrapHandler(gzipUserDetailsHandler)).Name("user_profile_details").Methods("GET") Router.Handle("/user/{id}/{username}/edit", wrapHandler(gzipUserProfileFormHandler)).Name("user_profile_edit").Methods("POST") - Router.Handle("/mod", gzipIndexModPanel).Name("mod_index") - Router.Handle("/mod/torrents", gzipTorrentsListPanel).Name("mod_tlist") - Router.Handle("/mod/torrents/{page}", gzipTorrentsListPanel).Name("mod_tlist_page") - Router.Handle("/mod/reports", gzipTorrentReportListPanel).Name("mod_trlist") - Router.Handle("/mod/reports/{page}", gzipTorrentReportListPanel).Name("mod_trlist_page") - Router.Handle("/mod/users", gzipUsersListPanel).Name("mod_ulist") - Router.Handle("/mod/users/{page}", gzipUsersListPanel).Name("mod_ulist_page") - Router.Handle("/mod/comments", gzipCommentsListPanel).Name("mod_clist") - Router.Handle("/mod/comments/{page}", gzipCommentsListPanel).Name("mod_clist_page") - Router.Handle("/mod/comment", gzipCommentsListPanel).Name("mod_cedit") // TODO - Router.Handle("/mod/torrent/", gzipTorrentEditModPanel).Name("mod_tedit").Methods("GET") - Router.Handle("/mod/torrent/", gzipTorrentPostEditModPanel).Name("mod_ptedit").Methods("POST") - Router.Handle("/mod/torrent/delete", gzipTorrentDeleteModPanel).Name("mod_tdelete") - Router.Handle("/mod/report/delete", gzipTorrentReportDeleteModPanel).Name("mod_trdelete") - Router.Handle("/mod/comment/delete", gzipCommentDeleteModPanel).Name("mod_cdelete") + Router.HandleFunc("/mod", IndexModPanel).Name("mod_index") + Router.HandleFunc("/mod/torrents", TorrentsListPanel).Name("mod_tlist") + Router.HandleFunc("/mod/torrents/{page}", TorrentsListPanel).Name("mod_tlist_page") + Router.HandleFunc("/mod/reports", TorrentReportListPanel).Name("mod_trlist") + Router.HandleFunc("/mod/reports/{page}", TorrentReportListPanel).Name("mod_trlist_page") + Router.HandleFunc("/mod/users", UsersListPanel).Name("mod_ulist") + Router.HandleFunc("/mod/users/{page}", UsersListPanel).Name("mod_ulist_page") + Router.HandleFunc("/mod/comments", CommentsListPanel).Name("mod_clist") + Router.HandleFunc("/mod/comments/{page}", CommentsListPanel).Name("mod_clist_page") + Router.HandleFunc("/mod/comment", CommentsListPanel).Name("mod_cedit") // TODO + Router.HandleFunc("/mod/torrent/", TorrentEditModPanel).Name("mod_tedit").Methods("GET") + Router.HandleFunc("/mod/torrent/", TorrentPostEditModPanel).Name("mod_ptedit").Methods("POST") + Router.HandleFunc("/mod/torrent/delete", TorrentDeleteModPanel).Name("mod_tdelete") + Router.HandleFunc("/mod/report/delete", TorrentReportDeleteModPanel).Name("mod_trdelete") + Router.HandleFunc("/mod/comment/delete", CommentDeleteModPanel).Name("mod_cdelete") //reporting a torrent Router.HandleFunc("/report/{id}", ReportTorrentHandler).Methods("POST").Name("post_comment") Router.PathPrefix("/captcha").Methods("GET").HandlerFunc(captcha.ServeFiles) - //Router.Handle("/report/create", gzipTorrentReportCreateHandler).Name("torrent_report_create").Methods("POST") + //Router.HandleFunc("/report/create", gzipTorrentReportCreateHandler).Name("torrent_report_create").Methods("POST") // TODO Allow only moderators to access /moderation/* - //Router.Handle("/moderation/report/delete", gzipTorrentReportDeleteHandler).Name("torrent_report_delete").Methods("POST") - //Router.Handle("/moderation/torrent/delete", gzipTorrentDeleteHandler).Name("torrent_delete").Methods("POST") + //Router.HandleFunc("/moderation/report/delete", gzipTorrentReportDeleteHandler).Name("torrent_report_delete").Methods("POST") + //Router.HandleFunc("/moderation/torrent/delete", gzipTorrentDeleteHandler).Name("torrent_delete").Methods("POST") Router.NotFoundHandler = http.HandlerFunc(NotFoundHandler) } From b12e812b36989f1ed526f770ad4887d8b628625b Mon Sep 17 00:00:00 2001 From: Atvaark Date: Thu, 11 May 2017 08:01:12 +0200 Subject: [PATCH 22/27] Add database logmode to the config This allows users to change the default logging verbosity (errors) to either *detailed* (prints SQL statements) or *silent*. Also added support for using a custom logger function. - Fixed the gorm unit test that checks the automigrations They will actually fail if any errors were logged now. - Added a postgres unit test Currently disabled because it would need a running local postgres db and a change to the .travis.yml file to work inside the CI build. --- .travis.yml | 4 +-- config/config.go | 23 +++++++++++++++++- db/gorm.go | 30 ++++++++++++++++++++++- db/gorm_test.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++-- main.go | 2 +- 5 files changed, 115 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5b8db6c5..c0a274f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,9 @@ before_install: script: # Downloads deps automatically. No need to add manually. - go list -f '{{.Deps}}' | tr "[" " " | tr "]" " " | xargs go list -e -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | grep -v 'github.com/ewhal/nyaa' | xargs go get -v -- go get +- go get - go build -- go vet +- go vet - go test -v ./... before_deploy: - ./package.sh diff --git a/config/config.go b/config/config.go index ae599c23..19b446d9 100644 --- a/config/config.go +++ b/config/config.go @@ -22,6 +22,7 @@ type Config struct { // DBParams will be directly passed to Gorm, and its internal // structure depends on the dialect for each db type DBParams string `json:"db_params"` + DBLogMode string `json:"db_logmode"` // tracker scraper config (required) Scrape ScraperConfig `json:"scraper"` // cache config @@ -32,7 +33,7 @@ type Config struct { I2P *I2PConfig `json:"i2p"` } -var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", DefaultScraperConfig, DefaultCacheConfig, DefaultSearchConfig, nil} +var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", "default", DefaultScraperConfig, DefaultCacheConfig, DefaultSearchConfig, nil} var allowedDatabaseTypes = map[string]bool{ "sqlite3": true, @@ -41,12 +42,19 @@ var allowedDatabaseTypes = map[string]bool{ "mssql": true, } +var allowedDBLogModes = map[string]bool{ + "default": true, // errors only + "detailed": true, + "silent": true, +} + func New() *Config { var config Config config.Host = Defaults.Host config.Port = Defaults.Port config.DBType = Defaults.DBType config.DBParams = Defaults.DBParams + config.DBLogMode = Defaults.DBLogMode config.Scrape = Defaults.Scrape config.Cache = Defaults.Cache return &config @@ -60,6 +68,7 @@ func (config *Config) BindFlags() func() error { host := flag.String("host", Defaults.Host, "binding address of the server") port := flag.Int("port", Defaults.Port, "port of the server") dbParams := flag.String("dbparams", Defaults.DBParams, "parameters to open the database (see Gorm's doc)") + dbLogMode := flag.String("dblogmode", Defaults.DBLogMode, "database log verbosity (errors only by default)") return func() error { // You can override fields in the config file with flags. @@ -70,6 +79,10 @@ func (config *Config) BindFlags() func() error { if err != nil { return err } + err = config.SetDBLogMode(*dbLogMode) + if err != nil { + return err + } err = config.HandleConfFileFlag(*confFile) return err } @@ -98,6 +111,14 @@ func (config *Config) SetDBType(db_type string) error { return nil } +func (config *Config) SetDBLogMode(db_logmode string) error { + if !allowedDBLogModes[db_logmode] { + return fmt.Errorf("unknown database log mode '%s'", db_logmode) + } + config.DBLogMode = db_logmode + return nil +} + func (config *Config) Read(input io.Reader) error { return json.NewDecoder(input).Decode(config) } diff --git a/db/gorm.go b/db/gorm.go index 77fa33a6..c2e75e60 100644 --- a/db/gorm.go +++ b/db/gorm.go @@ -9,12 +9,20 @@ import ( _ "github.com/jinzhu/gorm/dialects/sqlite" ) +type Logger interface { + Print(v ...interface{}) +} + +// use the default gorm logger that prints to stdout +var DefaultLogger Logger = nil + var ORM *gorm.DB var IsSqlite bool // GormInit init gorm ORM. -func GormInit(conf *config.Config) (*gorm.DB, error) { +func GormInit(conf *config.Config, logger Logger) (*gorm.DB, error) { + db, openErr := gorm.Open(conf.DBType, conf.DBParams) if openErr != nil { log.CheckError(openErr) @@ -35,9 +43,29 @@ func GormInit(conf *config.Config) (*gorm.DB, error) { db.LogMode(true) } + switch conf.DBLogMode { + case "detailed": + db.LogMode(true) + case "silent": + db.LogMode(false) + } + + if logger != nil { + db.SetLogger(logger) + } + db.AutoMigrate(&model.User{}, &model.UserFollows{}, &model.UserUploadsOld{}) + if db.Error != nil { + return db, db.Error + } db.AutoMigrate(&model.Torrent{}, &model.TorrentReport{}) + if db.Error != nil { + return db, db.Error + } db.AutoMigrate(&model.Comment{}, &model.OldComment{}) + if db.Error != nil { + return db, db.Error + } return db, nil } diff --git a/db/gorm_test.go b/db/gorm_test.go index eefc8fbb..ec61824a 100644 --- a/db/gorm_test.go +++ b/db/gorm_test.go @@ -1,22 +1,81 @@ package db import ( + "fmt" "testing" + "github.com/azhao12345/gorm" "github.com/ewhal/nyaa/config" ) -func TestGormInit(t *testing.T) { +type errorLogger struct { + t *testing.T +} + +func (logger *errorLogger) Print(values ...interface{}) { + if len(values) > 1 { + message := gorm.LogFormatter(values...) + level := values[0] + if level == "log" { + logger.t.Error(message...) + } + + fmt.Println(message...) + } +} + +func TestGormInitSqlite(t *testing.T) { conf := config.New() conf.DBType = "sqlite3" conf.DBParams = ":memory:?cache=shared&mode=memory" + conf.DBLogMode = "detailed" - db, err := GormInit(conf) + db, err := GormInit(conf, &errorLogger{t}) if err != nil { t.Errorf("failed to initialize database: %v", err) return } + if db == nil { + return + } + + err = db.Close() + if err != nil { + t.Errorf("failed to close database: %v", err) + } +} + +// This test requires a running postgres instance. To run it in CI build add these settings in the .travis.yml +// services: +// - postgresql +// before_script: +// - psql -c "CREATE DATABASE nyaapantsu;" -U postgres +// - psql -c "CREATE USER nyaapantsu WITH PASSWORD 'nyaapantsu';" -U postgres +// +// Then enable the test by setting this variable to "true" via ldflags: +// go test ./... -v -ldflags="-X github.com/ewhal/nyaa/db.testPostgres=true" +var testPostgres = "false" + +func TestGormInitPostgres(t *testing.T) { + if testPostgres != "true" { + t.Skip("skip", testPostgres) + } + + conf := config.New() + conf.DBType = "postgres" + conf.DBParams = "host=localhost user=nyaapantsu dbname=nyaapantsu sslmode=disable password=nyaapantsu" + conf.DBLogMode = "detailed" + + db, err := GormInit(conf, &errorLogger{t}) + if err != nil { + t.Errorf("failed to initialize database: %v", err) + } + + if db == nil { + return + } + err = db.Close() if err != nil { t.Errorf("failed to close database: %v", err) diff --git a/main.go b/main.go index 07260f53..48368b9e 100644 --- a/main.go +++ b/main.go @@ -118,7 +118,7 @@ func main() { if err != nil { log.CheckError(err) } - db.ORM, err = db.GormInit(conf) + db.ORM, err = db.GormInit(conf, db.DefaultLogger) if err != nil { log.Fatal(err.Error()) } From 3fc4236a7619071c7a891517b42963b32cb1f9f7 Mon Sep 17 00:00:00 2001 From: akuma06 Date: Thu, 11 May 2017 22:54:42 +0200 Subject: [PATCH 23/27] slight design improvement for template --- templates/view.html | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/templates/view.html b/templates/view.html index bc21d0c3..b96da65c 100644 --- a/templates/view.html +++ b/templates/view.html @@ -5,37 +5,43 @@ {{with .Torrent}}
-
-

{{.Name}}

-
+
+
+
+
+
+
+
+

Uploaded by {{.UploaderName}}

-
-
-
-
- +
+ Download! {{if ne .TorrentLink ""}} - + Torrent file {{end}} - + Report! {{ if HasAdmin $.User}} {{end}} - - +
+
+
+
+

{{T "description"}}

+
{{.Description}}
@@ -73,12 +79,6 @@
-
-
-

{{T "description"}}

-
{{.Description}}
-
-

{{T "comments"}}

From 019f37dc4d2fbab65decfa1374f09aa4f67046ce Mon Sep 17 00:00:00 2001 From: sfan5 Date: Thu, 11 May 2017 23:18:56 +0200 Subject: [PATCH 24/27] Stateless cookies (#372) * Remove eddie4.nl tracker It resolves to the same IP address as leechers-paradise which we already have. * Remove database usage from cookie auth * Hide "Remember Me" as it doesn't work yet --- config/trackers.go | 1 - model/user.go | 7 +- service/user/cookieHelper.go | 153 ++++++++++++++++++----------------- service/user/user.go | 26 +----- templates/FAQ.html | 1 - templates/user/login.html | 6 +- 6 files changed, 92 insertions(+), 102 deletions(-) diff --git a/config/trackers.go b/config/trackers.go index 8fdbb9b7..4f38177f 100644 --- a/config/trackers.go +++ b/config/trackers.go @@ -10,6 +10,5 @@ var Trackers = []string{ "udp://explodie.org:6969", "udp://tracker.opentrackr.org:1337", "udp://tracker.internetwarriors.net:1337/announce", - "udp://eddie4.nl:6969/announce", "http://mgtracker.org:6969/announce", "http://tracker.baka-sub.cf/announce"} diff --git a/model/user.go b/model/user.go index 1e1e3ca7..bf1fd654 100644 --- a/model/user.go +++ b/model/user.go @@ -12,8 +12,9 @@ type User struct { Status int `gorm:"column:status"` CreatedAt time.Time `gorm:"column:created_at"` UpdatedAt time.Time `gorm:"column:updated_at"` - Token string `gorm:"column:api_token"` - TokenExpiration time.Time `gorm:"column:api_token_expiry"` + // Currently unused (auth is stateless now) + /*Token string `gorm:"column:api_token"` + TokenExpiration time.Time `gorm:"column:api_token_expiry"`*/ Language string `gorm:"column:language"` // TODO: move this to PublicUser @@ -42,7 +43,7 @@ func (u User) Size() (s int) { 4*3 + //time.Time 3*2 + // arrays // string arrays - len(u.Username) + len(u.Password) + len(u.Email) + len(u.Token) + len(u.MD5) + len(u.Language) + len(u.Username) + len(u.Password) + len(u.Email) + len(u.MD5) + len(u.Language) s *= 8 // Ignoring foreign key users. Fuck them. diff --git a/service/user/cookieHelper.go b/service/user/cookieHelper.go index b28e5a98..940d5b3d 100644 --- a/service/user/cookieHelper.go +++ b/service/user/cookieHelper.go @@ -5,96 +5,98 @@ import ( "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/model" formStruct "github.com/ewhal/nyaa/service/user/form" - "github.com/ewhal/nyaa/util/log" "github.com/ewhal/nyaa/util/modelHelper" + "github.com/ewhal/nyaa/util/timeHelper" "github.com/gorilla/securecookie" "golang.org/x/crypto/bcrypt" "net/http" + "strconv" + "time" ) +const CookieName = "session" + +// If you want to keep login cookies between restarts you need to make these permanent var cookieHandler = securecookie.New( securecookie.GenerateRandomKey(64), securecookie.GenerateRandomKey(32)) -func Token(r *http.Request) (string, error) { - var token string - cookie, err := r.Cookie("session") +// Encoding & Decoding of the cookie value +func DecodeCookie(cookie_value string) (uint, error) { + value := make(map[string]string) + err := cookieHandler.Decode(CookieName, cookie_value, &value) if err != nil { - return token, err + return 0, err } - cookieValue := make(map[string]string) - err = cookieHandler.Decode("session", cookie.Value, &cookieValue) - if err != nil { - return token, err + time_int, _ := strconv.ParseInt(value["t"], 10, 0) + if timeHelper.IsExpired(time.Unix(time_int, 0)) { + return 0, errors.New("Cookie is expired") } - token = cookieValue["token"] - if len(token) == 0 { - return token, errors.New("token is empty") - } - return token, nil + ret, err := strconv.ParseUint(value["u"], 10, 0) + return uint(ret), err } -// SetCookie sets a cookie. -func SetCookie(w http.ResponseWriter, token string) (int, error) { +func EncodeCookie(user_id uint) (string, error) { + validUntil := timeHelper.FewDaysLater(7) // 1 week value := map[string]string{ - "token": token, + "u": strconv.FormatUint(uint64(user_id), 10), + "t": strconv.FormatInt(validUntil.Unix(), 10), } - encoded, err := cookieHandler.Encode("session", value) - if err != nil { - return http.StatusInternalServerError, err - } - cookie := &http.Cookie{ - Name: "session", - Value: encoded, - Path: "/", - } - http.SetCookie(w, cookie) - return http.StatusOK, nil + return cookieHandler.Encode(CookieName, value) } -// ClearCookie clears a cookie. func ClearCookie(w http.ResponseWriter) (int, error) { cookie := &http.Cookie{ - Name: "session", + Name: CookieName, Value: "", Path: "/", + HttpOnly: true, MaxAge: -1, } http.SetCookie(w, cookie) return http.StatusOK, nil } -// SetCookieHandler sets a cookie with email and password. +// SetCookieHandler sets the authentication cookie func SetCookieHandler(w http.ResponseWriter, email string, pass string) (int, error) { - if email != "" && pass != "" { - var user model.User - isValidEmail, _ := formStruct.EmailValidation(email, formStruct.NewErrors()) - if isValidEmail { - log.Debug("User entered valid email.") - if db.ORM.Where("email = ?", email).First(&user).RecordNotFound() { - return http.StatusNotFound, errors.New("User not found") - } - } else { - log.Debug("User entered username.") - if db.ORM.Where("username = ?", email).First(&user).RecordNotFound() { - return http.StatusNotFound, errors.New("User not found") - } - } - err := bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(pass)) - if err != nil { - return http.StatusUnauthorized, errors.New("Password incorrect") - } - if user.Status == -1 { - return http.StatusUnauthorized, errors.New("Account banned") - } - status, err := SetCookie(w, user.Token) - if err != nil { - return status, err - } - w.Header().Set("X-Auth-Token", user.Token) - return http.StatusOK, nil + if email == "" || pass == "" { + return http.StatusNotFound, errors.New("No username/password entered") } - return http.StatusNotFound, errors.New("user not found") + + var user model.User + // search by email or username + isValidEmail, _ := formStruct.EmailValidation(email, formStruct.NewErrors()) + if isValidEmail { + if db.ORM.Where("email = ?", email).First(&user).RecordNotFound() { + return http.StatusNotFound, errors.New("User not found") + } + } else { + if db.ORM.Where("username = ?", email).First(&user).RecordNotFound() { + return http.StatusNotFound, errors.New("User not found") + } + } + err := bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(pass)) + if err != nil { + return http.StatusUnauthorized, errors.New("Password incorrect") + } + if user.Status == -1 { + return http.StatusUnauthorized, errors.New("Account banned") + } + + encoded, err := EncodeCookie(user.ID) + if err != nil { + return http.StatusInternalServerError, err + } + cookie := &http.Cookie{ + Name: CookieName, + Value: encoded, + Path: "/", + HttpOnly: true, + } + http.SetCookie(w, cookie) + // also set response header for convenience + w.Header().Set("X-Auth-Token", encoded) + return http.StatusOK, nil } // RegisterHanderFromForm sets cookie from a RegistrationForm. @@ -111,24 +113,31 @@ func RegisterHandler(w http.ResponseWriter, r *http.Request) (int, error) { return RegisterHanderFromForm(w, registrationForm) } -// CurrentUser get a current user. +// CurrentUser determines the current user from the request func CurrentUser(r *http.Request) (model.User, error) { var user model.User - var token string - var err error - token = r.Header.Get("X-Auth-Token") - if len(token) > 0 { - log.Debug("header token exists") - } else { - token, err = Token(r) - log.Debug("header token does not exist") + var encoded string + + encoded = r.Header.Get("X-Auth-Token") + if len(encoded) == 0 { + // check cookie instead + cookie, err := r.Cookie(CookieName) if err != nil { return user, err } + encoded = cookie.Value } - if db.ORM.Where("api_token = ?", token).First(&user).RecordNotFound() { - return user, errors.New("user not found") + user_id, err := DecodeCookie(encoded) + if err != nil { + return user, err } - err = db.ORM.Model(&user).Error - return user, err + if db.ORM.Where("user_id = ?", user_id).First(&user).RecordNotFound() { + return user, errors.New("User not found") + } + + if user.Status == -1 { + // recheck as user might've been banned in the meantime + return user, errors.New("Account banned") + } + return user, nil } diff --git a/service/user/user.go b/service/user/user.go index 88b2ada6..27100c40 100644 --- a/service/user/user.go +++ b/service/user/user.go @@ -7,7 +7,6 @@ import ( "strconv" "time" - "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/model" formStruct "github.com/ewhal/nyaa/service/user/form" @@ -15,7 +14,6 @@ import ( "github.com/ewhal/nyaa/util/crypto" "github.com/ewhal/nyaa/util/log" "github.com/ewhal/nyaa/util/modelHelper" - "github.com/ewhal/nyaa/util/timeHelper" "golang.org/x/crypto/bcrypt" ) @@ -69,15 +67,8 @@ func CreateUserFromForm(registrationForm formStruct.RegistrationForm) (model.Use return user, err } } - token, err := crypto.GenerateRandomToken32() - if err != nil { - return user, errors.New("token not generated") - } user.Email = "" // unset email because it will be verified later - user.Token = token - user.TokenExpiration = timeHelper.FewDaysLater(config.AuthTokenExpirationDay) - log.Debugf("user %+v\n", user) if db.ORM.Create(&user).Error != nil { return user, errors.New("user not created") } @@ -157,17 +148,13 @@ func UpdateUserCore(user *model.User) (int, error) { } } - token, err := crypto.GenerateRandomToken32() + user.UpdatedAt = time.Now() + err := db.ORM.Save(user).Error if err != nil { return http.StatusInternalServerError, err } - user.Token = token - user.TokenExpiration = timeHelper.FewDaysLater(config.AuthTokenExpirationDay) - if db.ORM.Save(user).Error != nil { - return http.StatusInternalServerError, err - } - user.UpdatedAt = time.Now() + return http.StatusOK, nil } @@ -197,18 +184,13 @@ func UpdateUser(w http.ResponseWriter, form *formStruct.UserForm, currentUser *m form.Username = user.Username } if (form.Email != user.Email) { + // send verification to new email and keep old SendVerificationToUser(user, form.Email) form.Email = user.Email } log.Debugf("form %+v\n", form) modelHelper.AssignValue(&user, form) status, err := UpdateUserCore(&user) - if err != nil { - return user, status, err - } - if userPermission.CurrentUserIdentical(currentUser, user.ID) { - status, err = SetCookie(w, user.Token) - } return user, status, err } diff --git a/templates/FAQ.html b/templates/FAQ.html index b0a58191..f511f968 100644 --- a/templates/FAQ.html +++ b/templates/FAQ.html @@ -45,7 +45,6 @@ udp://tracker.leechers-paradise.org:6969 udp://explodie.org:6969 udp://tracker.opentrackr.org:1337 udp://tracker.internetwarriors.net:1337/announce -udp://eddie4.nl:6969/announce http://mgtracker.org:6969/announce http://tracker.baka-sub.cf/announce diff --git a/templates/user/login.html b/templates/user/login.html index 7ac16973..d720459d 100644 --- a/templates/user/login.html +++ b/templates/user/login.html @@ -1,4 +1,4 @@ -{{define "title"}}{{ T "sign_in_title" }}{{end}} +{{define "title"}}{{ T "sign_in_title" }}{{end}} {{define "contclass"}}cont-view{{end}} {{define "content"}}
@@ -24,8 +24,8 @@ {{end}}
- - + {{ T "forgot_password"}}
From a96902a7377d2ebbddfa0720350a1a5ebd4b729b Mon Sep 17 00:00:00 2001 From: sfan5 Date: Thu, 11 May 2017 23:31:34 +0200 Subject: [PATCH 25/27] Fix minor display bug on profile edit page --- router/userHandler.go | 10 +++++----- templates/_profile_edit.html | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/router/userHandler.go b/router/userHandler.go index e534edb4..5370896c 100755 --- a/router/userHandler.go +++ b/router/userHandler.go @@ -156,19 +156,19 @@ func UserProfileFormHandler(w http.ResponseWriter, r *http.Request) { if len(err) == 0 { modelHelper.BindValueForm(&b, r) if !userPermission.HasAdmin(currentUser) { - b.Username = currentUser.Username - b.Status = currentUser.Status + b.Username = userProfile.Username + b.Status = userProfile.Status } else { - if b.Status == 2 { + if userProfile.Status != b.Status && b.Status == 2 { err["errors"] = append(err["errors"], "Elevating status to moderator is prohibited") } } err = modelHelper.ValidateForm(&b, err) if len(err) == 0 { - if b.Email != currentUser.Email { + if b.Email != userProfile.Email { userService.SendVerificationToUser(*currentUser, b.Email) infos["infos"] = append(infos["infos"], fmt.Sprintf(T("email_changed"), b.Email)) - b.Email = currentUser.Email // reset, it will be set when user clicks verification + b.Email = userProfile.Email // reset, it will be set when user clicks verification } userProfile, _, errorUser = userService.UpdateUser(w, &b, currentUser, id) if errorUser != nil { diff --git a/templates/_profile_edit.html b/templates/_profile_edit.html index d549fd80..fa560084 100644 --- a/templates/_profile_edit.html +++ b/templates/_profile_edit.html @@ -81,8 +81,10 @@
{{ range (index $.FormErrors "status")}} From b6b2d836c0381cec13d9171585573a82faeee667 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Thu, 11 May 2017 23:35:51 +0200 Subject: [PATCH 26/27] Fix dates in RSS feed --- router/rssHandler.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/router/rssHandler.go b/router/rssHandler.go index c88a16fd..d5ba112f 100644 --- a/router/rssHandler.go +++ b/router/rssHandler.go @@ -26,22 +26,20 @@ func RSSHandler(w http.ResponseWriter, r *http.Request) { Link: &feeds.Link{Href: "https://" + config.WebAddress + "/"}, Created: createdAsTime, } - feed.Items = []*feeds.Item{} feed.Items = make([]*feeds.Item, len(torrents)) - for i := range torrents { - torrentJSON := torrents[i].ToJSON() + for i, torrent := range torrents { + torrentJSON := torrent.ToJSON() feed.Items[i] = &feeds.Item{ - // need a torrent view first Id: "https://" + config.WebAddress + "/view/" + strconv.FormatUint(uint64(torrents[i].ID), 10), - Title: torrents[i].Name, + Title: torrent.Name, Link: &feeds.Link{Href: string(torrentJSON.Magnet)}, Description: string(torrentJSON.Description), - Created: torrents[0].Date, - Updated: torrents[0].Date, + Created: torrent.Date, + Updated: torrent.Date, } } - //allow cross domain AJAX requests + // allow cross domain AJAX requests w.Header().Set("Access-Control-Allow-Origin", "*") rss, rssErr := feed.ToRss() if rssErr != nil { From b0982f07a63749f5fc1e8c78c3dcab98945a86de Mon Sep 17 00:00:00 2001 From: tomleb Date: Thu, 11 May 2017 17:54:53 -0400 Subject: [PATCH 27/27] Change color for bg-danger and fix night theme flickering issue (#370) * Change color for bg-danger * Fix annoying flickering with night theme * use new class instead of overriding bootstrap's --- public/css/style-night.css | 7 ++++++- public/css/style.css | 5 +++++ public/js/main.js | 9 ++------- templates/_profile_edit.html | 14 +++++++------- templates/admin_index.html | 14 ++++++++++++-- templates/index.html | 14 +++++++++++++- templates/user/login.html | 4 ++-- templates/user/register.html | 10 +++++----- 8 files changed, 52 insertions(+), 25 deletions(-) diff --git a/public/css/style-night.css b/public/css/style-night.css index 4209c410..01e46b5a 100644 --- a/public/css/style-night.css +++ b/public/css/style-night.css @@ -143,4 +143,9 @@ a:hover { } .modal-content .close { color: #fff; -} \ No newline at end of file +} + +.text-error { + background: #29363d; + color: #cf9fff; +} diff --git a/public/css/style.css b/public/css/style.css index db605fde..4df26ba4 100644 --- a/public/css/style.css +++ b/public/css/style.css @@ -383,3 +383,8 @@ footer { font-size: smaller; width: auto; /* Undo bootstrap's fixed width */ } + +.text-error { + background: white; + color: #cf9fff; +} diff --git a/public/js/main.js b/public/js/main.js index 1cc1c7b7..5d985acf 100644 --- a/public/js/main.js +++ b/public/js/main.js @@ -1,15 +1,10 @@ -// Night mode var night = localStorage.getItem("night"); -if (night == "true") { - $("head").append(''); -} - function toggleNightMode() { var night = localStorage.getItem("night"); if(night == "true") { - $("#style-dark")[0].remove() + document.getElementById("style-dark").remove() } else { - $("head").append(''); + document.getElementsByTagName("head")[0].append(darkStyleLink); } localStorage.setItem("night", (night == "true") ? "false" : "true"); } diff --git a/templates/_profile_edit.html b/templates/_profile_edit.html index fa560084..0493ad72 100644 --- a/templates/_profile_edit.html +++ b/templates/_profile_edit.html @@ -14,7 +14,7 @@
{{ range (index $.FormErrors "email")}} -

{{ . }}

+

{{ . }}

{{end}}
@@ -30,7 +30,7 @@
{{ range (index $.FormErrors "language")}} -

{{ . }}

+

{{ . }}

{{end}}
@@ -40,7 +40,7 @@
{{ range (index $.FormErrors "current_password")}} -

{{ . }}

+

{{ . }}

{{end}}
@@ -50,7 +50,7 @@
{{ range (index $.FormErrors "password")}} -

{{ . }}

+

{{ . }}

{{end}}
@@ -59,7 +59,7 @@
{{ range (index $.FormErrors "password_confirmation")}} -

{{ . }}

+

{{ . }}

{{end}}
@@ -70,7 +70,7 @@
{{ range (index $.FormErrors "username")}} -

{{ . }}

+

{{ . }}

{{end}}
@@ -88,7 +88,7 @@ {{ range (index $.FormErrors "status")}} -

{{ . }}

+

{{ . }}

{{end}} diff --git a/templates/admin_index.html b/templates/admin_index.html index 3943f9a3..11cb46c6 100644 --- a/templates/admin_index.html +++ b/templates/admin_index.html @@ -26,7 +26,18 @@ - +