From 6bf7ee3438d29ca6f155ec6b61230085b29da325 Mon Sep 17 00:00:00 2001 From: bakape Date: Wed, 10 May 2017 12:03:49 +0300 Subject: [PATCH] Search query caching --- cache/cache.go | 24 ++--- common/search.go | 46 +++++++++ router/templateVariables.go | 20 ++-- util/search/search.go | 185 ++++++++++++++---------------------- 4 files changed, 140 insertions(+), 135 deletions(-) create mode 100644 common/search.go diff --git a/cache/cache.go b/cache/cache.go index 640f0fef..0655b5a5 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -5,12 +5,12 @@ import ( "sync" "time" + "github.com/ewhal/nyaa/common" "github.com/ewhal/nyaa/model" - "github.com/ewhal/nyaa/util/search" ) var ( - cache = make(map[search.SearchParam]*list.Element, 10) + cache = make(map[common.SearchParam]*list.Element, 10) ll = list.New() totalUsed int mu sync.Mutex @@ -31,18 +31,17 @@ type Key struct { // Single cache entry type store struct { - // Controls general access to the contents of the struct, except for size - sync.Mutex + sync.Mutex // Controls general access to the contents of the struct lastFetched time.Time - key search.SearchParam - data []model.Torrents + key common.SearchParam + data []model.Torrent size int } // Check the cache for and existing record. If miss, run fn to retrieve fresh // values. -func CheckCache(key search.SearchParam, fn func() ([]model.Torrents, error)) ( - []model.Torrents, error, +func Get(key common.SearchParam, fn func() ([]model.Torrent, error)) ( + []model.Torrent, error, ) { s := getStore(key) @@ -59,12 +58,12 @@ func CheckCache(key search.SearchParam, fn func() ([]model.Torrents, error)) ( if err != nil { return nil, err } - s.lastFetched = time.Now() + s.update(data) return data, nil } // Retrieve a store from the cache or create a new one -func getStore(k search.SearchParam) (s *store) { +func getStore(k common.SearchParam) (s *store) { mu.Lock() defer mu.Unlock() @@ -85,7 +84,7 @@ func Clear() { defer mu.Unlock() ll = list.New() - cache = make(map[search.SearchParam]*list.Element, 10) + cache = make(map[common.SearchParam]*list.Element, 10) } // Update the total used memory counter and evict, if over limit @@ -109,7 +108,7 @@ func (s *store) isFresh() bool { // Stores the new values of s. Calculates and stores the new size. Passes the // delta to the central cache to fire eviction checks. -func (s *store) update(data []model.Torrents) { +func (s *store) update(data []model.Torrent) { newSize := 0 for _, d := range data { newSize += d.Size() @@ -117,6 +116,7 @@ func (s *store) update(data []model.Torrents) { s.data = data delta := newSize - s.size s.size = newSize + s.lastFetched = time.Now() // Technically it is possible to update the size even when the store is // already evicted, but that should never happen, unless you have a very diff --git a/common/search.go b/common/search.go new file mode 100644 index 00000000..214d21a1 --- /dev/null +++ b/common/search.go @@ -0,0 +1,46 @@ +package common + +import "strconv" + +type Status uint8 + +const ( + ShowAll Status = iota + FilterRemakes + Trusted + APlus +) + +type SortMode uint8 + +const ( + ID SortMode = iota + Name + Date + Downloads + Size +) + +type Category struct { + Main, Sub uint8 +} + +func (c Category) String() (s string) { + if c.Main != 0 { + s += strconv.Itoa(int(c.Main)) + } + s += "_" + if c.Sub != 0 { + s += strconv.Itoa(int(c.Sub)) + } + return +} + +type SearchParam struct { + Order bool // True means acsending + Status Status + Sort SortMode + Category Category + Max uint + Query string +} diff --git a/router/templateVariables.go b/router/templateVariables.go index aade2f1d..aa92dc9e 100644 --- a/router/templateVariables.go +++ b/router/templateVariables.go @@ -4,11 +4,11 @@ import ( "net/http" "net/url" + "github.com/ewhal/nyaa/common" "github.com/ewhal/nyaa/model" "github.com/ewhal/nyaa/service/captcha" "github.com/ewhal/nyaa/service/user" userForms "github.com/ewhal/nyaa/service/user/form" - "github.com/ewhal/nyaa/util/search" "github.com/gorilla/mux" ) @@ -87,7 +87,7 @@ type UserLoginFormVariables struct { type UserProfileVariables struct { UserProfile *model.User - FormInfos map[string][]string + FormInfos map[string][]string Search SearchForm Navigation Navigation User *model.User @@ -114,22 +114,22 @@ type UploadTemplateVariables struct { } type PanelIndexVbs struct { - Torrents []model.Torrent - Users []model.User - Comments []model.Comment + Torrents []model.Torrent + Users []model.User + Comments []model.Comment } type PanelTorrentListVbs struct { - Torrents []model.Torrent + Torrents []model.Torrent } type PanelUserListVbs struct { - Users []model.User + Users []model.User } type PanelCommentListVbs struct { - Comments []model.Comment + Comments []model.Comment } type PanelTorrentEdVbs struct { - Torrent model.Torrent + Torrent model.Torrent } /* @@ -143,7 +143,7 @@ type Navigation struct { } type SearchForm struct { - search.SearchParam + common.SearchParam Category string HideAdvancedSearch bool } diff --git a/util/search/search.go b/util/search/search.go index a1e35913..0a5872c1 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -1,71 +1,33 @@ package search import ( - "github.com/ewhal/nyaa/db" - "github.com/ewhal/nyaa/model" - "github.com/ewhal/nyaa/service/torrent" - "github.com/ewhal/nyaa/util/log" "net/http" "strconv" "strings" "unicode" "unicode/utf8" + + "github.com/ewhal/nyaa/cache" + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/db" + "github.com/ewhal/nyaa/model" + "github.com/ewhal/nyaa/service/torrent" + "github.com/ewhal/nyaa/util/log" ) -type Status uint8 - -const ( - ShowAll Status = iota - FilterRemakes - Trusted - APlus -) - -type SortMode uint8 - -const ( - ID SortMode = iota - Name - Date - Downloads - Size -) - -type Category struct { - Main, Sub uint8 -} - -func (c Category) String() (s string) { - if c.Main != 0 { - s += strconv.Itoa(int(c.Main)) - } - s += "_" - if c.Sub != 0 { - s += strconv.Itoa(int(c.Sub)) - } - return -} - -type SearchParam struct { - Order bool // True means acsending - Status Status - Sort SortMode - Category Category - Max uint - Query string -} - -func SearchByQuery(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, count int, err error) { +func SearchByQuery(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, count int, err error) { search, tor, count, err = searchByQuery(r, pagenum, true) return } -func SearchByQueryNoCount(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, err error) { +func SearchByQueryNoCount(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, err error) { search, tor, _, err = searchByQuery(r, pagenum, false) return } -func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchParam, tor []model.Torrent, count int, err error) { +func searchByQuery(r *http.Request, pagenum int, countAll bool) ( + search common.SearchParam, tor []model.Torrent, count int, err error, +) { max, err := strconv.ParseUint(r.URL.Query().Get("max"), 10, 32) if err != nil { max = 50 // default Value maxPerPage @@ -78,11 +40,11 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa switch s := r.URL.Query().Get("s"); s { case "1": - search.Status = FilterRemakes + search.Status = common.FilterRemakes case "2": - search.Status = Trusted + search.Status = common.Trusted case "3": - search.Status = APlus + search.Status = common.APlus } catString := r.URL.Query().Get("c") @@ -107,16 +69,16 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa switch s := r.URL.Query().Get("sort"); s { case "1": - search.Sort = Name + search.Sort = common.Name orderBy += "torrent_name" case "2": - search.Sort = Date + search.Sort = common.Date orderBy += "date" case "3": - search.Sort = Downloads + search.Sort = common.Downloads orderBy += "downloads" case "4": - search.Sort = Size + search.Sort = common.Size orderBy += "filesize" default: orderBy += "torrent_id" @@ -132,66 +94,63 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa orderBy += "desc" } - userID := r.URL.Query().Get("userID") + tor, err = cache.Get(search, func() (tor []model.Torrent, err error) { + parameters := torrentService.WhereParams{ + Params: make([]interface{}, 0, 64), + } + conditions := make([]string, 0, 64) + if search.Category.Main != 0 { + conditions = append(conditions, "category = ?") + parameters.Params = append(parameters.Params, string(catString[0])) + } + if search.Category.Sub != 0 { + conditions = append(conditions, "sub_category = ?") + parameters.Params = append(parameters.Params, string(catString[2])) + } + if search.Status != 0 { + if search.Status == 3 { + conditions = append(conditions, "status != ?") + } else { + conditions = append(conditions, "status = ?") + } + parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1)) + } - parameters := torrentService.WhereParams{ - Params: make([]interface{}, 0, 64), - } - conditions := make([]string, 0, 64) - if search.Category.Main != 0 { - conditions = append(conditions, "category = ?") - parameters.Params = append(parameters.Params, string(catString[0])) - } - if search.Category.Sub != 0 { - conditions = append(conditions, "sub_category = ?") - parameters.Params = append(parameters.Params, string(catString[2])) - } - if userID != "" { - conditions = append(conditions, "uploader = ?") - parameters.Params = append(parameters.Params, userID) - } - if search.Status != 0 { - if search.Status == 3 { - conditions = append(conditions, "status != ?") + searchQuerySplit := strings.Fields(search.Query) + for i, word := range searchQuerySplit { + firstRune, _ := utf8.DecodeRuneInString(word) + if len(word) == 1 && unicode.IsPunct(firstRune) { + // some queries have a single punctuation character + // which causes a full scan instead of using the index + // and yields no meaningful results. + // due to len() == 1 we're just looking at 1-byte/ascii + // punctuation characters. + continue + } + + // SQLite has case-insensitive LIKE, but no ILIKE + var operator string + if db.ORM.Dialect().GetName() == "sqlite3" { + operator = "LIKE ?" + } else { + operator = "ILIKE ?" + } + + // TODO: make this faster ? + conditions = append(conditions, "torrent_name "+operator) + parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") + } + + parameters.Conditions = strings.Join(conditions[:], " AND ") + log.Infof("SQL query is :: %s\n", parameters.Conditions) + if countAll { + tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(pagenum-1)) } else { - conditions = append(conditions, "status = ?") - } - parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1)) - } - - searchQuerySplit := strings.Fields(search.Query) - for i, word := range searchQuerySplit { - firstRune, _ := utf8.DecodeRuneInString(word) - if len(word) == 1 && unicode.IsPunct(firstRune) { - // some queries have a single punctuation character - // which causes a full scan instead of using the index - // and yields no meaningful results. - // due to len() == 1 we're just looking at 1-byte/ascii - // punctuation characters. - continue + tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(pagenum-1)) } - // TEMP: Workaround to at least make SQLite search testable for - // development. - // TODO: Actual case-insensitive search for SQLite - var operator string - if db.ORM.Dialect().GetName() == "sqlite3" { - operator = "LIKE ?" - } else { - operator = "ILIKE ?" - } + return + }) - // TODO: make this faster ? - conditions = append(conditions, "torrent_name "+operator) - parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") - } - - parameters.Conditions = strings.Join(conditions[:], " AND ") - log.Infof("SQL query is :: %s\n", parameters.Conditions) - if countAll { - tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(pagenum-1)) - } else { - tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(pagenum-1)) - } return }