Albirew/nyaa-pantsu
Albirew/nyaa-pantsu
Archivé
1
0
Bifurcation 0
Code Activité

Search query caching

Parcourir la source
Cette révision appartient à :
bakape 2017-05-10 12:03:49 +03:00
Parent 8b56f2e76e
révision 6bf7ee3438
4 fichiers modifiés avec 140 ajouts et 135 suppressions
Voir les statistiques Télécharger le patch en tant que fichier Télécharger le fichier diff Développer tous les fichiers Réduire tous les fichiers

24
cache/cache.go externe
Voir le fichier

@ -5,12 +5,12 @@ import (
"sync"
"time"
"github.com/ewhal/nyaa/common"
"github.com/ewhal/nyaa/model"
"github.com/ewhal/nyaa/util/search"
)
var (
cache = make(map[search.SearchParam]*list.Element, 10)
cache = make(map[common.SearchParam]*list.Element, 10)
ll = list.New()
totalUsed int
mu sync.Mutex
@ -31,18 +31,17 @@ type Key struct {
// Single cache entry
type store struct {
// Controls general access to the contents of the struct, except for size
sync.Mutex
sync.Mutex // Controls general access to the contents of the struct
lastFetched time.Time
key search.SearchParam
data []model.Torrents
key common.SearchParam
data []model.Torrent
size int
}
// Check the cache for and existing record. If miss, run fn to retrieve fresh
// values.
func CheckCache(key search.SearchParam, fn func() ([]model.Torrents, error)) (
[]model.Torrents, error,
func Get(key common.SearchParam, fn func() ([]model.Torrent, error)) (
[]model.Torrent, error,
) {
s := getStore(key)
@ -59,12 +58,12 @@ func CheckCache(key search.SearchParam, fn func() ([]model.Torrents, error)) (
if err != nil {
return nil, err
}
s.lastFetched = time.Now()
s.update(data)
return data, nil
}
// Retrieve a store from the cache or create a new one
func getStore(k search.SearchParam) (s *store) {
func getStore(k common.SearchParam) (s *store) {
mu.Lock()
defer mu.Unlock()
@ -85,7 +84,7 @@ func Clear() {
defer mu.Unlock()
ll = list.New()
cache = make(map[search.SearchParam]*list.Element, 10)
cache = make(map[common.SearchParam]*list.Element, 10)
}
// Update the total used memory counter and evict, if over limit
@ -109,7 +108,7 @@ func (s *store) isFresh() bool {
// Stores the new values of s. Calculates and stores the new size. Passes the
// delta to the central cache to fire eviction checks.
func (s *store) update(data []model.Torrents) {
func (s *store) update(data []model.Torrent) {
newSize := 0
for _, d := range data {
newSize += d.Size()
@ -117,6 +116,7 @@ func (s *store) update(data []model.Torrents) {
s.data = data
delta := newSize - s.size
s.size = newSize
s.lastFetched = time.Now()
// Technically it is possible to update the size even when the store is
// already evicted, but that should never happen, unless you have a very

46
common/search.go Fichier normal
Voir le fichier

@ -0,0 +1,46 @@
package common
import "strconv"
type Status uint8
const (
ShowAll Status = iota
FilterRemakes
Trusted
APlus
)
type SortMode uint8
const (
ID SortMode = iota
Name
Date
Downloads
Size
)
type Category struct {
Main, Sub uint8
}
func (c Category) String() (s string) {
if c.Main != 0 {
s += strconv.Itoa(int(c.Main))
}
s += "_"
if c.Sub != 0 {
s += strconv.Itoa(int(c.Sub))
}
return
}
type SearchParam struct {
Order bool // True means acsending
Status Status
Sort SortMode
Category Category
Max uint
Query string
}

20
router/templateVariables.go
Voir le fichier

@ -4,11 +4,11 @@ import (
"net/http"
"net/url"
"github.com/ewhal/nyaa/common"
"github.com/ewhal/nyaa/model"
"github.com/ewhal/nyaa/service/captcha"
"github.com/ewhal/nyaa/service/user"
userForms "github.com/ewhal/nyaa/service/user/form"
"github.com/ewhal/nyaa/util/search"
"github.com/gorilla/mux"
)
@ -87,7 +87,7 @@ type UserLoginFormVariables struct {
type UserProfileVariables struct {
UserProfile *model.User
FormInfos map[string][]string
FormInfos map[string][]string
Search SearchForm
Navigation Navigation
User *model.User
@ -114,22 +114,22 @@ type UploadTemplateVariables struct {
}
type PanelIndexVbs struct {
Torrents []model.Torrent
Users []model.User
Comments []model.Comment
Torrents []model.Torrent
Users []model.User
Comments []model.Comment
}
type PanelTorrentListVbs struct {
Torrents []model.Torrent
Torrents []model.Torrent
}
type PanelUserListVbs struct {
Users []model.User
Users []model.User
}
type PanelCommentListVbs struct {
Comments []model.Comment
Comments []model.Comment
}
type PanelTorrentEdVbs struct {
Torrent model.Torrent
Torrent model.Torrent
}
/*
@ -143,7 +143,7 @@ type Navigation struct {
}
type SearchForm struct {
search.SearchParam
common.SearchParam
Category string
HideAdvancedSearch bool
}

185
util/search/search.go
Voir le fichier

@ -1,71 +1,33 @@
package search
import (
"github.com/ewhal/nyaa/db"
"github.com/ewhal/nyaa/model"
"github.com/ewhal/nyaa/service/torrent"
"github.com/ewhal/nyaa/util/log"
"net/http"
"strconv"
"strings"
"unicode"
"unicode/utf8"
"github.com/ewhal/nyaa/cache"
"github.com/ewhal/nyaa/common"
"github.com/ewhal/nyaa/db"
"github.com/ewhal/nyaa/model"
"github.com/ewhal/nyaa/service/torrent"
"github.com/ewhal/nyaa/util/log"
)
type Status uint8
const (
ShowAll Status = iota
FilterRemakes
Trusted
APlus
)
type SortMode uint8
const (
ID SortMode = iota
Name
Date
Downloads
Size
)
type Category struct {
Main, Sub uint8
}
func (c Category) String() (s string) {
if c.Main != 0 {
s += strconv.Itoa(int(c.Main))
}
s += "_"
if c.Sub != 0 {
s += strconv.Itoa(int(c.Sub))
}
return
}
type SearchParam struct {
Order bool // True means acsending
Status Status
Sort SortMode
Category Category
Max uint
Query string
}
func SearchByQuery(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, count int, err error) {
func SearchByQuery(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, count int, err error) {
search, tor, count, err = searchByQuery(r, pagenum, true)
return
}
func SearchByQueryNoCount(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, err error) {
func SearchByQueryNoCount(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, err error) {
search, tor, _, err = searchByQuery(r, pagenum, false)
return
}
func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchParam, tor []model.Torrent, count int, err error) {
func searchByQuery(r *http.Request, pagenum int, countAll bool) (
search common.SearchParam, tor []model.Torrent, count int, err error,
) {
max, err := strconv.ParseUint(r.URL.Query().Get("max"), 10, 32)
if err != nil {
max = 50 // default Value maxPerPage
@ -78,11 +40,11 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
switch s := r.URL.Query().Get("s"); s {
case "1":
search.Status = FilterRemakes
search.Status = common.FilterRemakes
case "2":
search.Status = Trusted
search.Status = common.Trusted
case "3":
search.Status = APlus
search.Status = common.APlus
}
catString := r.URL.Query().Get("c")
@ -107,16 +69,16 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
switch s := r.URL.Query().Get("sort"); s {
case "1":
search.Sort = Name
search.Sort = common.Name
orderBy += "torrent_name"
case "2":
search.Sort = Date
search.Sort = common.Date
orderBy += "date"
case "3":
search.Sort = Downloads
search.Sort = common.Downloads
orderBy += "downloads"
case "4":
search.Sort = Size
search.Sort = common.Size
orderBy += "filesize"
default:
orderBy += "torrent_id"
@ -132,66 +94,63 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
orderBy += "desc"
}
userID := r.URL.Query().Get("userID")
tor, err = cache.Get(search, func() (tor []model.Torrent, err error) {
parameters := torrentService.WhereParams{
Params: make([]interface{}, 0, 64),
}
conditions := make([]string, 0, 64)
if search.Category.Main != 0 {
conditions = append(conditions, "category = ?")
parameters.Params = append(parameters.Params, string(catString[0]))
}
if search.Category.Sub != 0 {
conditions = append(conditions, "sub_category = ?")
parameters.Params = append(parameters.Params, string(catString[2]))
}
if search.Status != 0 {
if search.Status == 3 {
conditions = append(conditions, "status != ?")
} else {
conditions = append(conditions, "status = ?")
}
parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1))
}
parameters := torrentService.WhereParams{
Params: make([]interface{}, 0, 64),
}
conditions := make([]string, 0, 64)
if search.Category.Main != 0 {
conditions = append(conditions, "category = ?")
parameters.Params = append(parameters.Params, string(catString[0]))
}
if search.Category.Sub != 0 {
conditions = append(conditions, "sub_category = ?")
parameters.Params = append(parameters.Params, string(catString[2]))
}
if userID != "" {
conditions = append(conditions, "uploader = ?")
parameters.Params = append(parameters.Params, userID)
}
if search.Status != 0 {
if search.Status == 3 {
conditions = append(conditions, "status != ?")
searchQuerySplit := strings.Fields(search.Query)
for i, word := range searchQuerySplit {
firstRune, _ := utf8.DecodeRuneInString(word)
if len(word) == 1 && unicode.IsPunct(firstRune) {
// some queries have a single punctuation character
// which causes a full scan instead of using the index
// and yields no meaningful results.
// due to len() == 1 we're just looking at 1-byte/ascii
// punctuation characters.
continue
}
// SQLite has case-insensitive LIKE, but no ILIKE
var operator string
if db.ORM.Dialect().GetName() == "sqlite3" {
operator = "LIKE ?"
} else {
operator = "ILIKE ?"
}
// TODO: make this faster ?
conditions = append(conditions, "torrent_name "+operator)
parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%")
}
parameters.Conditions = strings.Join(conditions[:], " AND ")
log.Infof("SQL query is :: %s\n", parameters.Conditions)
if countAll {
tor, count, err = torrentService.GetTorrentsOrderBy(&parameters, orderBy, int(search.Max), int(search.Max)*(pagenum-1))
} else {
conditions = append(conditions, "status = ?")
}
parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1))
}
searchQuerySplit := strings.Fields(search.Query)
for i, word := range searchQuerySplit {
firstRune, _ := utf8.DecodeRuneInString(word)
if len(word) == 1 && unicode.IsPunct(firstRune) {
// some queries have a single punctuation character
// which causes a full scan instead of using the index
// and yields no meaningful results.
// due to len() == 1 we're just looking at 1-byte/ascii
// punctuation characters.
continue
tor, err = torrentService.GetTorrentsOrderByNoCount(&parameters, orderBy, int(search.Max), int(search.Max)*(pagenum-1))
}
// TEMP: Workaround to at least make SQLite search testable for
// development.
// TODO: Actual case-insensitive search for SQLite
var operator string
if db.ORM.Dialect().GetName() == "sqlite3" {
operator = "LIKE ?"
} else {
operator = "ILIKE ?"
}
return
})
// TODO: make this faster ?
conditions = append(conditions, "torrent_name "+operator)
parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%")
}
parameters.Conditions = strings.Join(conditions[:], " AND ")
log.Infof("SQL query is :: %s\n", parameters.Conditions)
if countAll {
tor, count, err = torrentService.GetTorrentsOrderBy(&parameters, orderBy, int(search.Max), int(search.Max)*(pagenum-1))
} else {
tor, err = torrentService.GetTorrentsOrderByNoCount(&parameters, orderBy, int(search.Max), int(search.Max)*(pagenum-1))
}
return
}