Merge pull request #222 from bakape/search-caching

Search caching

cache/cache.go

@@ -0,0 +1,128 @@
+package cache
+
+import (
+	"container/list"
+	"sync"
+	"time"
+
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/model"
+)
+
+const expiryTime = time.Minute
+
+var (
+	cache     = make(map[common.SearchParam]*list.Element, 10)
+	ll        = list.New()
+	totalUsed int
+	mu        sync.Mutex
+
+	// Size sets the maximum size of the cache before evicting unread data in MB
+	Size float64 = 1 << 10
+)
+
+// Key stores the ID of either a thread or board page
+type Key struct {
+	LastN uint8
+	Board string
+	ID    uint64
+}
+
+// Single cache entry
+type store struct {
+	sync.Mutex  // Controls general access to the contents of the struct
+	lastFetched time.Time
+	key         common.SearchParam
+	data        []model.Torrent
+	count, size int
+}
+
+// Check the cache for and existing record. If miss, run fn to retrieve fresh
+// values.
+func Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) (
+	data []model.Torrent, count int, err error,
+) {
+	s := getStore(key)
+
+	// Also keeps multiple requesters from simultaneously requesting the same
+	// data
+	s.Lock()
+	defer s.Unlock()
+
+	if s.isFresh() {
+		return s.data, s.count, nil
+	}
+
+	data, count, err = fn()
+	if err != nil {
+		return
+	}
+	s.update(data, count)
+	return
+}
+
+// Retrieve a store from the cache or create a new one
+func getStore(k common.SearchParam) (s *store) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	el := cache[k]
+	if el == nil {
+		s = &store{key: k}
+		cache[k] = ll.PushFront(s)
+	} else {
+		ll.MoveToFront(el)
+		s = el.Value.(*store)
+	}
+	return s
+}
+
+// Clear the cache. Only used for testing.
+func Clear() {
+	mu.Lock()
+	defer mu.Unlock()
+
+	ll = list.New()
+	cache = make(map[common.SearchParam]*list.Element, 10)
+}
+
+// Update the total used memory counter and evict, if over limit
+func updateUsedSize(delta int) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	totalUsed += delta
+
+	for totalUsed > int(Size)<<20 {
+		s := ll.Remove(ll.Back()).(*store)
+		delete(cache, s.key)
+		totalUsed -= s.size
+	}
+}
+
+// Return, if the data can still be considered fresh, without querying the DB
+func (s *store) isFresh() bool {
+	if s.lastFetched.IsZero() { // New store
+		return false
+	}
+	return s.lastFetched.Add(expiryTime).After(time.Now())
+}
+
+// Stores the new values of s. Calculates and stores the new size. Passes the
+// delta to the central cache to fire eviction checks.
+func (s *store) update(data []model.Torrent, count int) {
+	newSize := 0
+	for _, d := range data {
+		newSize += d.Size()
+	}
+	s.data = data
+	s.count = count
+	delta := newSize - s.size
+	s.size = newSize
+	s.lastFetched = time.Now()
+
+	// Technically it is possible to update the size even when the store is
+	// already evicted, but that should never happen, unless you have a very
+	// small cache, very large stored datasets and a lot of traffic.
+	updateUsedSize(delta)
+}

common/search.go

@@ -0,0 +1,47 @@
+package common
+
+import "strconv"
+
+type Status uint8
+
+const (
+	ShowAll Status = iota
+	FilterRemakes
+	Trusted
+	APlus
+)
+
+type SortMode uint8
+
+const (
+	ID SortMode = iota
+	Name
+	Date
+	Downloads
+	Size
+)
+
+type Category struct {
+	Main, Sub uint8
+}
+
+func (c Category) String() (s string) {
+	if c.Main != 0 {
+		s += strconv.Itoa(int(c.Main))
+	}
+	s += "_"
+	if c.Sub != 0 {
+		s += strconv.Itoa(int(c.Sub))
+	}
+	return
+}
+
+type SearchParam struct {
+	Order    bool // True means acsending
+	Status   Status
+	Sort     SortMode
+	Category Category
+	Page     int
+	Max      uint
+	Query    string
+}

main.go

@@ -3,20 +3,19 @@ package main
 import (
 	"bufio"
 	"flag"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
 
-	"github.com/nicksnyder/go-i18n/i18n"
-
+	"github.com/ewhal/nyaa/cache"
 	"github.com/ewhal/nyaa/config"
 	"github.com/ewhal/nyaa/db"
 	"github.com/ewhal/nyaa/network"
 	"github.com/ewhal/nyaa/router"
 	"github.com/ewhal/nyaa/util/log"
 	"github.com/ewhal/nyaa/util/signals"
-
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
+	"github.com/nicksnyder/go-i18n/i18n"
 )
 
 func initI18N() {
@@ -51,6 +50,7 @@ func main() {
 	conf := config.New()
 	processFlags := conf.BindFlags()
 	defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout")
+	flag.Float64Var(&cache.Size, "c", cache.Size, "size of the search cache in MB")
 	flag.Parse()
 	if *defaults {
 		stdout := bufio.NewWriter(os.Stdout)

model/comment.go

@@ -17,6 +17,11 @@ type Comment struct {
 	User    *User    `gorm:"ForeignKey:user_id"`
 }
 
+// Returns the total size of memory recursively allocated for this struct
+func (c Comment) Size() int {
+	return (3 + 3*3 + 2 + 2 + len(c.Content)) * 8
+}
+
 type OldComment struct {
 	TorrentID uint      `gorm:"column:torrent_id"`
 	Username  string    `gorm:"column:username"`
@@ -26,6 +31,11 @@ type OldComment struct {
 	Torrent *Torrent `gorm:"ForeignKey:torrent_id"`
 }
 
+// Returns the total size of memory recursively allocated for this struct
+func (c OldComment) Size() int {
+	return (1 + 2*2 + len(c.Username) + len(c.Content) + 3 + 1) * 8
+}
+
 func (c OldComment) TableName() string {
 	// cba to rename this in the db
 	// TODO: Update database schema to fix this hack

model/torrent.go

@@ -41,6 +41,31 @@ type Torrent struct {
 	Comments    []Comment    `gorm:"ForeignKey:torrent_id"`
 }
 
+// Returns the total size of memory recursively allocated for this struct
+func (t Torrent) Size() (s int) {
+	s += 8 + // ints
+		2*3 + // time.Time
+		2 + // pointers
+		4*2 + // string pointers
+		// string array sizes
+		len(t.Name) + len(t.Hash) + len(t.Description) + len(t.WebsiteLink) +
+		2*2 // array pointers
+	s *= 8 // Assume 64 bit OS
+
+	if t.Uploader != nil {
+		s += t.Uploader.Size()
+	}
+	for _, c := range t.OldComments {
+		s += c.Size()
+	}
+	for _, c := range t.Comments {
+		s += c.Size()
+	}
+
+	return
+
+}
+
 // TODO Add field to specify kind of reports
 // TODO Add CreatedAt field
 // INFO User can be null (anonymous reports)

model/user.go

@@ -26,6 +26,21 @@ type User struct {
 	Torrents []Torrent `gorm:"ForeignKey:UploaderID"`
 }
 
+// Returns the total size of memory recursively allocated for this struct
+func (u User) Size() (s int) {
+	s += 4 + // ints
+		6*2 + // string pointers
+		4*3 + //time.Time
+		3*2 + // arrays
+		// string arrays
+		len(u.Username) + len(u.Password) + len(u.Email) + len(u.Token) + len(u.MD5) + len(u.Language)
+	s *= 8
+
+	// Ignoring foreign key users. Fuck them.
+
+	return
+}
+
 type PublicUser struct {
 	User *User
 }

router/homeHandler.go

@@ -1,15 +1,18 @@
 package router
 
 import (
+	"html"
+	"net/http"
+	"strconv"
+
+	"github.com/ewhal/nyaa/cache"
+	"github.com/ewhal/nyaa/common"
 	"github.com/ewhal/nyaa/model"
 	"github.com/ewhal/nyaa/service/torrent"
 	"github.com/ewhal/nyaa/util"
 	"github.com/ewhal/nyaa/util/languages"
 	"github.com/ewhal/nyaa/util/log"
 	"github.com/gorilla/mux"
-	"html"
-	"net/http"
-	"strconv"
 )
 
 func HomeHandler(w http.ResponseWriter, r *http.Request) {
@@ -36,11 +39,17 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	search := common.SearchParam{
+		Max:  uint(maxPerPage),
+		Page: pagenum,
+	}
+	torrents, nbTorrents, err := cache.Get(search, func() ([]model.Torrent, int, error) {
 		torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1))
 		if !log.CheckError(err) {
 			util.SendError(w, err, 400)
-		return
 		}
+		return torrents, nbTorrents, err
+	})
 
 	b := model.TorrentsToJSON(torrents)
 

router/templateVariables.go

@@ -4,11 +4,11 @@ import (
 	"net/http"
 	"net/url"
 
+	"github.com/ewhal/nyaa/common"
 	"github.com/ewhal/nyaa/model"
 	"github.com/ewhal/nyaa/service/captcha"
 	"github.com/ewhal/nyaa/service/user"
 	userForms "github.com/ewhal/nyaa/service/user/form"
-	"github.com/ewhal/nyaa/util/search"
 	"github.com/gorilla/mux"
 )
 
@@ -147,7 +147,7 @@ type Navigation struct {
 }
 
 type SearchForm struct {
-	search.SearchParam
+	common.SearchParam
 	Category           string
 	HideAdvancedSearch bool
 }

util/search/search.go

@@ -1,71 +1,33 @@
 package search
 
 import (
-	"github.com/ewhal/nyaa/db"
-	"github.com/ewhal/nyaa/model"
-	"github.com/ewhal/nyaa/service/torrent"
-	"github.com/ewhal/nyaa/util/log"
 	"net/http"
 	"strconv"
 	"strings"
 	"unicode"
 	"unicode/utf8"
+
+	"github.com/ewhal/nyaa/cache"
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/db"
+	"github.com/ewhal/nyaa/model"
+	"github.com/ewhal/nyaa/service/torrent"
+	"github.com/ewhal/nyaa/util/log"
 )
 
-type Status uint8
-
-const (
-	ShowAll Status = iota
-	FilterRemakes
-	Trusted
-	APlus
-)
-
-type SortMode uint8
-
-const (
-	ID SortMode = iota
-	Name
-	Date
-	Downloads
-	Size
-)
-
-type Category struct {
-	Main, Sub uint8
-}
-
-func (c Category) String() (s string) {
-	if c.Main != 0 {
-		s += strconv.Itoa(int(c.Main))
-	}
-	s += "_"
-	if c.Sub != 0 {
-		s += strconv.Itoa(int(c.Sub))
-	}
-	return
-}
-
-type SearchParam struct {
-	Order    bool // True means acsending
-	Status   Status
-	Sort     SortMode
-	Category Category
-	Max      uint
-	Query    string
-}
-
-func SearchByQuery(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, count int, err error) {
+func SearchByQuery(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, count int, err error) {
 	search, tor, count, err = searchByQuery(r, pagenum, true)
 	return
 }
 
-func SearchByQueryNoCount(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, err error) {
+func SearchByQueryNoCount(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, err error) {
 	search, tor, _, err = searchByQuery(r, pagenum, false)
 	return
 }
 
-func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchParam, tor []model.Torrent, count int, err error) {
+func searchByQuery(r *http.Request, pagenum int, countAll bool) (
+	search common.SearchParam, tor []model.Torrent, count int, err error,
+) {
 	max, err := strconv.ParseUint(r.URL.Query().Get("max"), 10, 32)
 	if err != nil {
 		max = 50 // default Value maxPerPage
@@ -74,15 +36,16 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
 	}
 	search.Max = uint(max)
 
+	search.Page = pagenum
 	search.Query = r.URL.Query().Get("q")
 
 	switch s := r.URL.Query().Get("s"); s {
 	case "1":
-		search.Status = FilterRemakes
+		search.Status = common.FilterRemakes
 	case "2":
-		search.Status = Trusted
+		search.Status = common.Trusted
 	case "3":
-		search.Status = APlus
+		search.Status = common.APlus
 	}
 
 	catString := r.URL.Query().Get("c")
@@ -107,16 +70,16 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
 
 	switch s := r.URL.Query().Get("sort"); s {
 	case "1":
-		search.Sort = Name
+		search.Sort = common.Name
 		orderBy += "torrent_name"
 	case "2":
-		search.Sort = Date
+		search.Sort = common.Date
 		orderBy += "date"
 	case "3":
-		search.Sort = Downloads
+		search.Sort = common.Downloads
 		orderBy += "downloads"
 	case "4":
-		search.Sort = Size
+		search.Sort = common.Size
 		orderBy += "filesize"
 	default:
 		orderBy += "torrent_id"
@@ -132,8 +95,7 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
 		orderBy += "desc"
 	}
 
-	userID := r.URL.Query().Get("userID")
-
+	tor, count, err = cache.Get(search, func() (tor []model.Torrent, count int, err error) {
 		parameters := torrentService.WhereParams{
 			Params: make([]interface{}, 0, 64),
 		}
@@ -146,10 +108,6 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
 			conditions = append(conditions, "sub_category = ?")
 			parameters.Params = append(parameters.Params, string(catString[2]))
 		}
-	if userID != "" {
-		conditions = append(conditions, "uploader = ?")
-		parameters.Params = append(parameters.Params, userID)
-	}
 		if search.Status != 0 {
 			if search.Status == 3 {
 				conditions = append(conditions, "status != ?")
@@ -171,9 +129,7 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
 				continue
 			}
 
-		// TEMP: Workaround to at least make SQLite search testable for
-		// development.
-		// TODO: Actual case-insensitive search for SQLite
+			// SQLite has case-insensitive LIKE, but no ILIKE
 			var operator string
 			if db.ORM.Dialect().GetName() == "sqlite3" {
 				operator = "LIKE ?"
@@ -189,9 +145,13 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
 		parameters.Conditions = strings.Join(conditions[:], " AND ")
 		log.Infof("SQL query is :: %s\n", parameters.Conditions)
 		if countAll {
-		tor, count, err = torrentService.GetTorrentsOrderBy(&parameters, orderBy, int(search.Max), int(search.Max)*(pagenum-1))
+			tor, count, err = torrentService.GetTorrentsOrderBy(&parameters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
 		} else {
-		tor, err = torrentService.GetTorrentsOrderByNoCount(&parameters, orderBy, int(search.Max), int(search.Max)*(pagenum-1))
+			tor, err = torrentService.GetTorrentsOrderByNoCount(&parameters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
 		}
+
+		return
+	})
+
 	return
 }