From 38f23a3d7f5fe1102783d0c93baec616f566e790 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 12 May 2017 17:41:26 +0200 Subject: [PATCH 1/4] Use ts_query on Postgres for performance reasons Untested. --- router/wrapHandler.go | 2 +- util/search/search.go | 35 ++++++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/router/wrapHandler.go b/router/wrapHandler.go index d0170f44..1f98325d 100644 --- a/router/wrapHandler.go +++ b/router/wrapHandler.go @@ -46,4 +46,4 @@ func (wh *wrappedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func wrapHandler(handler http.Handler) http.Handler { return &wrappedHandler{handler} -} \ No newline at end of file +} diff --git a/util/search/search.go b/util/search/search.go index 1b96529b..64e1d8fb 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -18,17 +18,29 @@ import ( ) var searchOperator string +var useTSQuery bool func Configure(conf *config.SearchConfig) (err error) { - // SQLite has case-insensitive LIKE, but no ILIKE - if db.ORM.Dialect().GetName() == "sqlite3" { - searchOperator = "LIKE ?" - } else { + // Postgres needs ILIKE for case-insensitivity + if db.ORM.Dialect().GetName() == "postgres" { searchOperator = "ILIKE ?" + useTSQuery = true + } else { + searchOperator = "LIKE ?" + useTSQuery = false } return } +func stringIsAscii(input string) bool { + for _, char := range input { + if char > 127 { + return false + } + } + return true +} + func SearchByQuery(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, count int, err error) { search, tor, count, err = searchByQuery(r, pagenum, true) return @@ -164,7 +176,7 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( } searchQuerySplit := strings.Fields(search.Query) - for i, word := range searchQuerySplit { + for _, word := range searchQuerySplit { firstRune, _ := utf8.DecodeRuneInString(word) if len(word) == 1 && unicode.IsPunct(firstRune) { // some queries have a single punctuation character @@ -175,9 +187,14 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( continue } - // TODO: make this faster ? - conditions = append(conditions, "torrent_name "+searchOperator) - parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") + if useTSQuery && stringIsAscii(word) { + conditions = append(conditions, "torrent_name @@ plainto_tsquery(?)") + parameters.Params = append(parameters.Params, word) + } else { + // TODO: possible to make this faster? + conditions = append(conditions, "torrent_name "+searchOperator) + parameters.Params = append(parameters.Params, "%"+word+"%") + } } parameters.Conditions = strings.Join(conditions[:], " AND ") @@ -194,4 +211,4 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( return }) return -} \ No newline at end of file +} From 5cb672b06ab4e5e7d6273bcccfc4caa1763aec27 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 12 May 2017 17:54:08 +0200 Subject: [PATCH 2/4] Stop RSS from being so slow --- router/rssHandler.go | 3 +-- service/torrent/torrent.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/router/rssHandler.go b/router/rssHandler.go index d5ba112f..799e8008 100644 --- a/router/rssHandler.go +++ b/router/rssHandler.go @@ -6,7 +6,6 @@ import ( "github.com/ewhal/nyaa/util/search" "github.com/gorilla/feeds" "net/http" - "strconv" "time" ) @@ -31,7 +30,7 @@ func RSSHandler(w http.ResponseWriter, r *http.Request) { for i, torrent := range torrents { torrentJSON := torrent.ToJSON() feed.Items[i] = &feeds.Item{ - Id: "https://" + config.WebAddress + "/view/" + strconv.FormatUint(uint64(torrents[i].ID), 10), + Id: "https://" + config.WebAddress + "/view/" + torrentJSON.ID, Title: torrent.Name, Link: &feeds.Link{Href: string(torrentJSON.Magnet)}, Description: string(torrentJSON.Description), diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go index caa776f7..bc19b2f9 100644 --- a/service/torrent/torrent.go +++ b/service/torrent/torrent.go @@ -129,7 +129,7 @@ func getTorrentsOrderBy(parameters *serviceBase.WhereParams, orderBy string, lim if conditions != "" { dbQuery = dbQuery + " WHERE " + conditions } - if strings.Contains(conditions, "torrent_name") { + if strings.Contains(conditions, "torrent_name") && offset > 0 { dbQuery = "WITH t AS (SELECT * FROM torrents WHERE " + conditions + ") SELECT * FROM t" } From eb10a9baa3afd1187d735cef9acca05a28692326 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 12 May 2017 17:56:22 +0200 Subject: [PATCH 3/4] Adjust read & write timeouts --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 48368b9e..5b56615a 100644 --- a/main.go +++ b/main.go @@ -38,8 +38,8 @@ func RunServer(conf *config.Config) { // Set up server, srv := &http.Server{ - WriteTimeout: 15 * time.Second, - ReadTimeout: 15 * time.Second, + WriteTimeout: 24 * time.Second, + ReadTimeout: 8 * time.Second, } l, err := network.CreateHTTPListener(conf) log.CheckError(err) From c5fe70800d0d8468804a52ed01f142a71e2b28ab Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 12 May 2017 19:38:08 +0200 Subject: [PATCH 4/4] Disable CTEs and TSQuery for now --- service/torrent/torrent.go | 3 ++- util/search/search.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go index bc19b2f9..7c5eade4 100644 --- a/service/torrent/torrent.go +++ b/service/torrent/torrent.go @@ -129,9 +129,10 @@ func getTorrentsOrderBy(parameters *serviceBase.WhereParams, orderBy string, lim if conditions != "" { dbQuery = dbQuery + " WHERE " + conditions } + /* This makes all queries take roughly the same amount of time (lots)... if strings.Contains(conditions, "torrent_name") && offset > 0 { dbQuery = "WITH t AS (SELECT * FROM torrents WHERE " + conditions + ") SELECT * FROM t" - } + }*/ if orderBy == "" { // default OrderBy orderBy = "torrent_id DESC" diff --git a/util/search/search.go b/util/search/search.go index 64e1d8fb..a78fc9b7 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -21,13 +21,15 @@ var searchOperator string var useTSQuery bool func Configure(conf *config.SearchConfig) (err error) { + useTSQuery = false // Postgres needs ILIKE for case-insensitivity if db.ORM.Dialect().GetName() == "postgres" { searchOperator = "ILIKE ?" - useTSQuery = true + //useTSQuery = true + // !!DISABLED!! because this makes search a lot stricter + // (only matches at word borders) } else { searchOperator = "LIKE ?" - useTSQuery = false } return }