From 38f23a3d7f5fe1102783d0c93baec616f566e790 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 12 May 2017 17:41:26 +0200 Subject: [PATCH 01/11] Use ts_query on Postgres for performance reasons Untested. --- router/wrapHandler.go | 2 +- util/search/search.go | 35 ++++++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/router/wrapHandler.go b/router/wrapHandler.go index d0170f44..1f98325d 100644 --- a/router/wrapHandler.go +++ b/router/wrapHandler.go @@ -46,4 +46,4 @@ func (wh *wrappedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func wrapHandler(handler http.Handler) http.Handler { return &wrappedHandler{handler} -} \ No newline at end of file +} diff --git a/util/search/search.go b/util/search/search.go index 1b96529b..64e1d8fb 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -18,17 +18,29 @@ import ( ) var searchOperator string +var useTSQuery bool func Configure(conf *config.SearchConfig) (err error) { - // SQLite has case-insensitive LIKE, but no ILIKE - if db.ORM.Dialect().GetName() == "sqlite3" { - searchOperator = "LIKE ?" - } else { + // Postgres needs ILIKE for case-insensitivity + if db.ORM.Dialect().GetName() == "postgres" { searchOperator = "ILIKE ?" + useTSQuery = true + } else { + searchOperator = "LIKE ?" + useTSQuery = false } return } +func stringIsAscii(input string) bool { + for _, char := range input { + if char > 127 { + return false + } + } + return true +} + func SearchByQuery(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, count int, err error) { search, tor, count, err = searchByQuery(r, pagenum, true) return @@ -164,7 +176,7 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( } searchQuerySplit := strings.Fields(search.Query) - for i, word := range searchQuerySplit { + for _, word := range searchQuerySplit { firstRune, _ := utf8.DecodeRuneInString(word) if len(word) == 1 && unicode.IsPunct(firstRune) { // some queries have a single punctuation character @@ -175,9 +187,14 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( continue } - // TODO: make this faster ? - conditions = append(conditions, "torrent_name "+searchOperator) - parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") + if useTSQuery && stringIsAscii(word) { + conditions = append(conditions, "torrent_name @@ plainto_tsquery(?)") + parameters.Params = append(parameters.Params, word) + } else { + // TODO: possible to make this faster? + conditions = append(conditions, "torrent_name "+searchOperator) + parameters.Params = append(parameters.Params, "%"+word+"%") + } } parameters.Conditions = strings.Join(conditions[:], " AND ") @@ -194,4 +211,4 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( return }) return -} \ No newline at end of file +} From 5cb672b06ab4e5e7d6273bcccfc4caa1763aec27 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 12 May 2017 17:54:08 +0200 Subject: [PATCH 02/11] Stop RSS from being so slow --- router/rssHandler.go | 3 +-- service/torrent/torrent.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/router/rssHandler.go b/router/rssHandler.go index d5ba112f..799e8008 100644 --- a/router/rssHandler.go +++ b/router/rssHandler.go @@ -6,7 +6,6 @@ import ( "github.com/ewhal/nyaa/util/search" "github.com/gorilla/feeds" "net/http" - "strconv" "time" ) @@ -31,7 +30,7 @@ func RSSHandler(w http.ResponseWriter, r *http.Request) { for i, torrent := range torrents { torrentJSON := torrent.ToJSON() feed.Items[i] = &feeds.Item{ - Id: "https://" + config.WebAddress + "/view/" + strconv.FormatUint(uint64(torrents[i].ID), 10), + Id: "https://" + config.WebAddress + "/view/" + torrentJSON.ID, Title: torrent.Name, Link: &feeds.Link{Href: string(torrentJSON.Magnet)}, Description: string(torrentJSON.Description), diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go index caa776f7..bc19b2f9 100644 --- a/service/torrent/torrent.go +++ b/service/torrent/torrent.go @@ -129,7 +129,7 @@ func getTorrentsOrderBy(parameters *serviceBase.WhereParams, orderBy string, lim if conditions != "" { dbQuery = dbQuery + " WHERE " + conditions } - if strings.Contains(conditions, "torrent_name") { + if strings.Contains(conditions, "torrent_name") && offset > 0 { dbQuery = "WITH t AS (SELECT * FROM torrents WHERE " + conditions + ") SELECT * FROM t" } From eb10a9baa3afd1187d735cef9acca05a28692326 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 12 May 2017 17:56:22 +0200 Subject: [PATCH 03/11] Adjust read & write timeouts --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 48368b9e..5b56615a 100644 --- a/main.go +++ b/main.go @@ -38,8 +38,8 @@ func RunServer(conf *config.Config) { // Set up server, srv := &http.Server{ - WriteTimeout: 15 * time.Second, - ReadTimeout: 15 * time.Second, + WriteTimeout: 24 * time.Second, + ReadTimeout: 8 * time.Second, } l, err := network.CreateHTTPListener(conf) log.CheckError(err) From c5fe70800d0d8468804a52ed01f142a71e2b28ab Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 12 May 2017 19:38:08 +0200 Subject: [PATCH 04/11] Disable CTEs and TSQuery for now --- service/torrent/torrent.go | 3 ++- util/search/search.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go index bc19b2f9..7c5eade4 100644 --- a/service/torrent/torrent.go +++ b/service/torrent/torrent.go @@ -129,9 +129,10 @@ func getTorrentsOrderBy(parameters *serviceBase.WhereParams, orderBy string, lim if conditions != "" { dbQuery = dbQuery + " WHERE " + conditions } + /* This makes all queries take roughly the same amount of time (lots)... if strings.Contains(conditions, "torrent_name") && offset > 0 { dbQuery = "WITH t AS (SELECT * FROM torrents WHERE " + conditions + ") SELECT * FROM t" - } + }*/ if orderBy == "" { // default OrderBy orderBy = "torrent_id DESC" diff --git a/util/search/search.go b/util/search/search.go index 64e1d8fb..a78fc9b7 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -21,13 +21,15 @@ var searchOperator string var useTSQuery bool func Configure(conf *config.SearchConfig) (err error) { + useTSQuery = false // Postgres needs ILIKE for case-insensitivity if db.ORM.Dialect().GetName() == "postgres" { searchOperator = "ILIKE ?" - useTSQuery = true + //useTSQuery = true + // !!DISABLED!! because this makes search a lot stricter + // (only matches at word borders) } else { searchOperator = "LIKE ?" - useTSQuery = false } return } From 66c38c5cc742da9a67bb53a6fe8ffdb4f78c6f5a Mon Sep 17 00:00:00 2001 From: SpaceDustTeapot Date: Fri, 12 May 2017 23:28:03 +0100 Subject: [PATCH 05/11] Updated README.md. Stated the required Golang version Stated the required Golang version since ubuntu has 1.6 by default. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d5a109d..e26d1bd5 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ that anyone will be able to deploy locally or remotely. # Installation Ubuntu 17.04 fails to build, use a different OS or docker -* Install [Golang](https://golang.org/doc/install) +* Install [Golang](https://golang.org/doc/install) (version >=1.8) * `go get github.com/ewhal/nyaa` * `go build` * Download DB and place it in your root folder named as "nyaa.db" From 00bb382cdbede431837db5c8bbcfd0da28572e39 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Sat, 13 May 2017 00:31:27 +0200 Subject: [PATCH 06/11] Modpanel stuff (#401) * Fix table on modpanel index * Fix actions on torrent report list page * Fix error reporting on modpanel templates * Convenience functions for modpanel Link to view page primarily and to edit page as a seperate link * Missing confirm before torrent deletion * Add edit button directly onto view page --- router/modpanel.go | 6 ++++-- router/templateFunctions.go | 10 ++++++++++ templates/admin/panelindex.html | 19 +++++++++++++------ templates/admin/torrent_report.html | 6 +++--- templates/admin/torrentlist.html | 8 +++++--- templates/view.html | 1 + 6 files changed, 36 insertions(+), 14 deletions(-) diff --git a/router/modpanel.go b/router/modpanel.go index 61006e83..5446e00b 100644 --- a/router/modpanel.go +++ b/router/modpanel.go @@ -33,7 +33,8 @@ func IndexModPanel(w http.ResponseWriter, r *http.Request) { languages.SetTranslationFromRequest(panelIndex, r, "en-us") htv := PanelIndexVbs{torrents, model.TorrentReportsToJSON(torrentReports), users, comments, NewSearchForm(), currentUser, r.URL} - _ = panelIndex.ExecuteTemplate(w, "admin_index.html", htv) + err := panelIndex.ExecuteTemplate(w, "admin_index.html", htv) + log.CheckError(err) } else { http.Error(w, "admins only", http.StatusForbidden) } @@ -217,7 +218,8 @@ func TorrentPostEditModPanel(w http.ResponseWriter, r *http.Request) { } languages.SetTranslationFromRequest(panelTorrentEd, r, "en-us") htv := PanelTorrentEdVbs{uploadForm, NewSearchForm(), currentUser, err, infos, r.URL} - _ = panelTorrentEd.ExecuteTemplate(w, "admin_index.html", htv) + err_ := panelTorrentEd.ExecuteTemplate(w, "admin_index.html", htv) + log.CheckError(err_) } func CommentDeleteModPanel(w http.ResponseWriter, r *http.Request) { diff --git a/router/templateFunctions.go b/router/templateFunctions.go index 0946dce8..d1b3fe54 100644 --- a/router/templateFunctions.go +++ b/router/templateFunctions.go @@ -26,6 +26,16 @@ var FuncMap = template.FuncMap{ } return "error" }, + "genViewTorrentRoute": func(torrent_id uint) string { + // Helper for when you have an uint while genRoute("view_torrent", ...) takes a string + // FIXME better solution? + s := strconv.FormatUint(uint64(torrent_id), 10) + url, err := Router.Get("view_torrent").URL("id", s) + if err == nil { + return url.String() + } + return "error" + }, "genNav": func(nav Navigation, currentUrl *url.URL, pagesSelectable int) template.HTML { var ret = "" if (nav.TotalItem > 0) { diff --git a/templates/admin/panelindex.html b/templates/admin/panelindex.html index b718ffca..4385dba3 100644 --- a/templates/admin/panelindex.html +++ b/templates/admin/panelindex.html @@ -7,9 +7,12 @@ Uploader Action -{{ range .Torrents}} -{{ .Name }}{{ .UploaderID }} -{{ T "delete" }} +{{range .Torrents}} + + {{ .Name }} (Edit) + {{ .UploaderID }} + {{ T "delete" }} + {{end}}