Merge pull request #222 from bakape/search-caching

Search caching

cache/cache.go

@@ -0,0 +1,128 @@
+package cache
+
+import (
+	"container/list"
+	"sync"
+	"time"
+
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/model"
+)
+
+const expiryTime = time.Minute
+
+var (
+	cache     = make(map[common.SearchParam]*list.Element, 10)
+	ll        = list.New()
+	totalUsed int
+	mu        sync.Mutex
+
+	// Size sets the maximum size of the cache before evicting unread data in MB
+	Size float64 = 1 << 10
+)
+
+// Key stores the ID of either a thread or board page
+type Key struct {
+	LastN uint8
+	Board string
+	ID    uint64
+}
+
+// Single cache entry
+type store struct {
+	sync.Mutex  // Controls general access to the contents of the struct
+	lastFetched time.Time
+	key         common.SearchParam
+	data        []model.Torrent
+	count, size int
+}
+
+// Check the cache for and existing record. If miss, run fn to retrieve fresh
+// values.
+func Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) (
+	data []model.Torrent, count int, err error,
+) {
+	s := getStore(key)
+
+	// Also keeps multiple requesters from simultaneously requesting the same
+	// data
+	s.Lock()
+	defer s.Unlock()
+
+	if s.isFresh() {
+		return s.data, s.count, nil
+	}
+
+	data, count, err = fn()
+	if err != nil {
+		return
+	}
+	s.update(data, count)
+	return
+}
+
+// Retrieve a store from the cache or create a new one
+func getStore(k common.SearchParam) (s *store) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	el := cache[k]
+	if el == nil {
+		s = &store{key: k}
+		cache[k] = ll.PushFront(s)
+	} else {
+		ll.MoveToFront(el)
+		s = el.Value.(*store)
+	}
+	return s
+}
+
+// Clear the cache. Only used for testing.
+func Clear() {
+	mu.Lock()
+	defer mu.Unlock()
+
+	ll = list.New()
+	cache = make(map[common.SearchParam]*list.Element, 10)
+}
+
+// Update the total used memory counter and evict, if over limit
+func updateUsedSize(delta int) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	totalUsed += delta
+
+	for totalUsed > int(Size)<<20 {
+		s := ll.Remove(ll.Back()).(*store)
+		delete(cache, s.key)
+		totalUsed -= s.size
+	}
+}
+
+// Return, if the data can still be considered fresh, without querying the DB
+func (s *store) isFresh() bool {
+	if s.lastFetched.IsZero() { // New store
+		return false
+	}
+	return s.lastFetched.Add(expiryTime).After(time.Now())
+}
+
+// Stores the new values of s. Calculates and stores the new size. Passes the
+// delta to the central cache to fire eviction checks.
+func (s *store) update(data []model.Torrent, count int) {
+	newSize := 0
+	for _, d := range data {
+		newSize += d.Size()
+	}
+	s.data = data
+	s.count = count
+	delta := newSize - s.size
+	s.size = newSize
+	s.lastFetched = time.Now()
+
+	// Technically it is possible to update the size even when the store is
+	// already evicted, but that should never happen, unless you have a very
+	// small cache, very large stored datasets and a lot of traffic.
+	updateUsedSize(delta)
+}

common/search.go

@@ -0,0 +1,47 @@
+package common
+
+import "strconv"
+
+type Status uint8
+
+const (
+	ShowAll Status = iota
+	FilterRemakes
+	Trusted
+	APlus
+)
+
+type SortMode uint8
+
+const (
+	ID SortMode = iota
+	Name
+	Date
+	Downloads
+	Size
+)
+
+type Category struct {
+	Main, Sub uint8
+}
+
+func (c Category) String() (s string) {
+	if c.Main != 0 {
+		s += strconv.Itoa(int(c.Main))
+	}
+	s += "_"
+	if c.Sub != 0 {
+		s += strconv.Itoa(int(c.Sub))
+	}
+	return
+}
+
+type SearchParam struct {
+	Order    bool // True means acsending
+	Status   Status
+	Sort     SortMode
+	Category Category
+	Page     int
+	Max      uint
+	Query    string
+}

main.go

@@ -3,20 +3,19 @@ package main
 import (
 	"bufio"
 	"flag"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
 
-	"github.com/nicksnyder/go-i18n/i18n"
-
+	"github.com/ewhal/nyaa/cache"
 	"github.com/ewhal/nyaa/config"
 	"github.com/ewhal/nyaa/db"
 	"github.com/ewhal/nyaa/network"
 	"github.com/ewhal/nyaa/router"
 	"github.com/ewhal/nyaa/util/log"
 	"github.com/ewhal/nyaa/util/signals"
-
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
+	"github.com/nicksnyder/go-i18n/i18n"
 )
 
 func initI18N() {
@@ -51,6 +50,7 @@ func main() {
 	conf := config.New()
 	processFlags := conf.BindFlags()
 	defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout")
+	flag.Float64Var(&cache.Size, "c", cache.Size, "size of the search cache in MB")
 	flag.Parse()
 	if *defaults {
 		stdout := bufio.NewWriter(os.Stdout)

model/comment.go

@@ -17,6 +17,11 @@ type Comment struct {
 	User    *User    `gorm:"ForeignKey:user_id"`
 }
 
+// Returns the total size of memory recursively allocated for this struct
+func (c Comment) Size() int {
+	return (3 + 3*3 + 2 + 2 + len(c.Content)) * 8
+}
+
 type OldComment struct {
 	TorrentID uint      `gorm:"column:torrent_id"`
 	Username  string    `gorm:"column:username"`
@@ -26,6 +31,11 @@ type OldComment struct {
 	Torrent *Torrent `gorm:"ForeignKey:torrent_id"`
 }
 
+// Returns the total size of memory recursively allocated for this struct
+func (c OldComment) Size() int {
+	return (1 + 2*2 + len(c.Username) + len(c.Content) + 3 + 1) * 8
+}
+
 func (c OldComment) TableName() string {
 	// cba to rename this in the db
 	// TODO: Update database schema to fix this hack

model/torrent.go

@@ -34,24 +34,49 @@ type Torrent struct {
 	Filesize    int64     `gorm:"column:filesize"`
 	Description string    `gorm:"column:description"`
 	WebsiteLink string    `gorm:"column:website_link"`
-	DeletedAt *time.Time
+	DeletedAt   *time.Time
 
 	Uploader    *User        `gorm:"ForeignKey:UploaderId"`
 	OldComments []OldComment `gorm:"ForeignKey:torrent_id"`
 	Comments    []Comment    `gorm:"ForeignKey:torrent_id"`
 }
 
+// Returns the total size of memory recursively allocated for this struct
+func (t Torrent) Size() (s int) {
+	s += 8 + // ints
+		2*3 + // time.Time
+		2 + // pointers
+		4*2 + // string pointers
+		// string array sizes
+		len(t.Name) + len(t.Hash) + len(t.Description) + len(t.WebsiteLink) +
+		2*2 // array pointers
+	s *= 8 // Assume 64 bit OS
+
+	if t.Uploader != nil {
+		s += t.Uploader.Size()
+	}
+	for _, c := range t.OldComments {
+		s += c.Size()
+	}
+	for _, c := range t.Comments {
+		s += c.Size()
+	}
+
+	return
+
+}
+
 // TODO Add field to specify kind of reports
 // TODO Add CreatedAt field
 // INFO User can be null (anonymous reports)
 // FIXME  can't preload field Torrents for model.TorrentReport
 type TorrentReport struct {
-	ID          uint     `gorm:"column:torrent_report_id;primary_key"`
-	Description string   `gorm:"column:type"`
-	TorrentID  uint
-	UserID    uint
-	Torrent     Torrent  `gorm:"AssociationForeignKey:TorrentID;ForeignKey:ID"`
-	User        User     `gorm:"AssociationForeignKey:UserID;ForeignKey:ID"`
+	ID          uint   `gorm:"column:torrent_report_id;primary_key"`
+	Description string `gorm:"column:type"`
+	TorrentID   uint
+	UserID      uint
+	Torrent     Torrent `gorm:"AssociationForeignKey:TorrentID;ForeignKey:ID"`
+	User        User    `gorm:"AssociationForeignKey:UserID;ForeignKey:ID"`
 }
 
 /* We need a JSON object instead of a Gorm structure because magnet URLs are
@@ -89,9 +114,9 @@ type TorrentJSON struct {
 }
 
 type TorrentReportJson struct {
-	ID          uint     `json:"id"`
-	Description string   `json:"description"`
-	Torrent     TorrentJSON  `json:"torrent"`
+	ID          uint        `json:"id"`
+	Description string      `json:"description"`
+	Torrent     TorrentJSON `json:"torrent"`
 	User        string
 }
 

model/user.go

@@ -26,6 +26,21 @@ type User struct {
 	Torrents []Torrent `gorm:"ForeignKey:UploaderID"`
 }
 
+// Returns the total size of memory recursively allocated for this struct
+func (u User) Size() (s int) {
+	s += 4 + // ints
+		6*2 + // string pointers
+		4*3 + //time.Time
+		3*2 + // arrays
+		// string arrays
+		len(u.Username) + len(u.Password) + len(u.Email) + len(u.Token) + len(u.MD5) + len(u.Language)
+	s *= 8
+
+	// Ignoring foreign key users. Fuck them.
+
+	return
+}
+
 type PublicUser struct {
 	User *User
 }

router/homeHandler.go

@@ -1,15 +1,18 @@
 package router
 
 import (
+	"html"
+	"net/http"
+	"strconv"
+
+	"github.com/ewhal/nyaa/cache"
+	"github.com/ewhal/nyaa/common"
 	"github.com/ewhal/nyaa/model"
 	"github.com/ewhal/nyaa/service/torrent"
 	"github.com/ewhal/nyaa/util"
 	"github.com/ewhal/nyaa/util/languages"
 	"github.com/ewhal/nyaa/util/log"
 	"github.com/gorilla/mux"
-	"html"
-	"net/http"
-	"strconv"
 )
 
 func HomeHandler(w http.ResponseWriter, r *http.Request) {
@@ -36,11 +39,17 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1))
-	if !log.CheckError(err) {
-		util.SendError(w, err, 400)
-		return
+	search := common.SearchParam{
+		Max:  uint(maxPerPage),
+		Page: pagenum,
 	}
+	torrents, nbTorrents, err := cache.Get(search, func() ([]model.Torrent, int, error) {
+		torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1))
+		if !log.CheckError(err) {
+			util.SendError(w, err, 400)
+		}
+		return torrents, nbTorrents, err
+	})
 
 	b := model.TorrentsToJSON(torrents)
 

router/templateVariables.go

@@ -4,11 +4,11 @@ import (
 	"net/http"
 	"net/url"
 
+	"github.com/ewhal/nyaa/common"
 	"github.com/ewhal/nyaa/model"
 	"github.com/ewhal/nyaa/service/captcha"
 	"github.com/ewhal/nyaa/service/user"
 	userForms "github.com/ewhal/nyaa/service/user/form"
-	"github.com/ewhal/nyaa/util/search"
 	"github.com/gorilla/mux"
 )
 
@@ -87,7 +87,7 @@ type UserLoginFormVariables struct {
 
 type UserProfileVariables struct {
 	UserProfile *model.User
-	FormInfos        map[string][]string
+	FormInfos   map[string][]string
 	Search      SearchForm
 	Navigation  Navigation
 	User        *model.User
@@ -114,22 +114,22 @@ type UploadTemplateVariables struct {
 }
 
 type PanelIndexVbs struct {
-	Torrents  []model.Torrent
-	Users  []model.User
-	Comments  []model.Comment
+	Torrents []model.Torrent
+	Users    []model.User
+	Comments []model.Comment
 }
 
 type PanelTorrentListVbs struct {
-	Torrents  []model.Torrent
+	Torrents []model.Torrent
 }
 type PanelUserListVbs struct {
-	Users  []model.User
+	Users []model.User
 }
 type PanelCommentListVbs struct {
-	Comments  []model.Comment
+	Comments []model.Comment
 }
 type PanelTorrentEdVbs struct {
-	Torrent  model.Torrent
+	Torrent model.Torrent
 }
 
 type ViewTorrentReportsVariables struct {
@@ -147,7 +147,7 @@ type Navigation struct {
 }
 
 type SearchForm struct {
-	search.SearchParam
+	common.SearchParam
 	Category           string
 	HideAdvancedSearch bool
 }

util/search/search.go

@@ -1,71 +1,33 @@
 package search
 
 import (
-	"github.com/ewhal/nyaa/db"
-	"github.com/ewhal/nyaa/model"
-	"github.com/ewhal/nyaa/service/torrent"
-	"github.com/ewhal/nyaa/util/log"
 	"net/http"
 	"strconv"
 	"strings"
 	"unicode"
 	"unicode/utf8"
+
+	"github.com/ewhal/nyaa/cache"
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/db"
+	"github.com/ewhal/nyaa/model"
+	"github.com/ewhal/nyaa/service/torrent"
+	"github.com/ewhal/nyaa/util/log"
 )
 
-type Status uint8
-
-const (
-	ShowAll Status = iota
-	FilterRemakes
-	Trusted
-	APlus
-)
-
-type SortMode uint8
-
-const (
-	ID SortMode = iota
-	Name
-	Date
-	Downloads
-	Size
-)
-
-type Category struct {
-	Main, Sub uint8
-}
-
-func (c Category) String() (s string) {
-	if c.Main != 0 {
-		s += strconv.Itoa(int(c.Main))
-	}
-	s += "_"
-	if c.Sub != 0 {
-		s += strconv.Itoa(int(c.Sub))
-	}
-	return
-}
-
-type SearchParam struct {
-	Order    bool // True means acsending
-	Status   Status
-	Sort     SortMode
-	Category Category
-	Max      uint
-	Query    string
-}
-
-func SearchByQuery(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, count int, err error) {
+func SearchByQuery(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, count int, err error) {
 	search, tor, count, err = searchByQuery(r, pagenum, true)
 	return
 }
 
-func SearchByQueryNoCount(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, err error) {
+func SearchByQueryNoCount(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, err error) {
 	search, tor, _, err = searchByQuery(r, pagenum, false)
 	return
 }
 
-func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchParam, tor []model.Torrent, count int, err error) {
+func searchByQuery(r *http.Request, pagenum int, countAll bool) (
+	search common.SearchParam, tor []model.Torrent, count int, err error,
+) {
 	max, err := strconv.ParseUint(r.URL.Query().Get("max"), 10, 32)
 	if err != nil {
 		max = 50 // default Value maxPerPage
@@ -74,15 +36,16 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
 	}
 	search.Max = uint(max)
 
+	search.Page = pagenum
 	search.Query = r.URL.Query().Get("q")
 
 	switch s := r.URL.Query().Get("s"); s {
 	case "1":
-		search.Status = FilterRemakes
+		search.Status = common.FilterRemakes
 	case "2":
-		search.Status = Trusted
+		search.Status = common.Trusted
 	case "3":
-		search.Status = APlus
+		search.Status = common.APlus
 	}
 
 	catString := r.URL.Query().Get("c")
@@ -107,16 +70,16 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
 
 	switch s := r.URL.Query().Get("sort"); s {
 	case "1":
-		search.Sort = Name
+		search.Sort = common.Name
 		orderBy += "torrent_name"
 	case "2":
-		search.Sort = Date
+		search.Sort = common.Date
 		orderBy += "date"
 	case "3":
-		search.Sort = Downloads
+		search.Sort = common.Downloads
 		orderBy += "downloads"
 	case "4":
-		search.Sort = Size
+		search.Sort = common.Size
 		orderBy += "filesize"
 	default:
 		orderBy += "torrent_id"
@@ -132,66 +95,63 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa
 		orderBy += "desc"
 	}
 
-	userID := r.URL.Query().Get("userID")
+	tor, count, err = cache.Get(search, func() (tor []model.Torrent, count int, err error) {
+		parameters := torrentService.WhereParams{
+			Params: make([]interface{}, 0, 64),
+		}
+		conditions := make([]string, 0, 64)
+		if search.Category.Main != 0 {
+			conditions = append(conditions, "category = ?")
+			parameters.Params = append(parameters.Params, string(catString[0]))
+		}
+		if search.Category.Sub != 0 {
+			conditions = append(conditions, "sub_category = ?")
+			parameters.Params = append(parameters.Params, string(catString[2]))
+		}
+		if search.Status != 0 {
+			if search.Status == 3 {
+				conditions = append(conditions, "status != ?")
+			} else {
+				conditions = append(conditions, "status = ?")
+			}
+			parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1))
+		}
 
-	parameters := torrentService.WhereParams{
-		Params: make([]interface{}, 0, 64),
-	}
-	conditions := make([]string, 0, 64)
-	if search.Category.Main != 0 {
-		conditions = append(conditions, "category = ?")
-		parameters.Params = append(parameters.Params, string(catString[0]))
-	}
-	if search.Category.Sub != 0 {
-		conditions = append(conditions, "sub_category = ?")
-		parameters.Params = append(parameters.Params, string(catString[2]))
-	}
-	if userID != "" {
-		conditions = append(conditions, "uploader = ?")
-		parameters.Params = append(parameters.Params, userID)
-	}
-	if search.Status != 0 {
-		if search.Status == 3 {
-			conditions = append(conditions, "status != ?")
+		searchQuerySplit := strings.Fields(search.Query)
+		for i, word := range searchQuerySplit {
+			firstRune, _ := utf8.DecodeRuneInString(word)
+			if len(word) == 1 && unicode.IsPunct(firstRune) {
+				// some queries have a single punctuation character
+				// which causes a full scan instead of using the index
+				// and yields no meaningful results.
+				// due to len() == 1 we're just looking at 1-byte/ascii
+				// punctuation characters.
+				continue
+			}
+
+			// SQLite has case-insensitive LIKE, but no ILIKE
+			var operator string
+			if db.ORM.Dialect().GetName() == "sqlite3" {
+				operator = "LIKE ?"
+			} else {
+				operator = "ILIKE ?"
+			}
+
+			// TODO: make this faster ?
+			conditions = append(conditions, "torrent_name "+operator)
+			parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%")
+		}
+
+		parameters.Conditions = strings.Join(conditions[:], " AND ")
+		log.Infof("SQL query is :: %s\n", parameters.Conditions)
+		if countAll {
+			tor, count, err = torrentService.GetTorrentsOrderBy(&parameters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
 		} else {
-			conditions = append(conditions, "status = ?")
-		}
-		parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1))
-	}
-
-	searchQuerySplit := strings.Fields(search.Query)
-	for i, word := range searchQuerySplit {
-		firstRune, _ := utf8.DecodeRuneInString(word)
-		if len(word) == 1 && unicode.IsPunct(firstRune) {
-			// some queries have a single punctuation character
-			// which causes a full scan instead of using the index
-			// and yields no meaningful results.
-			// due to len() == 1 we're just looking at 1-byte/ascii
-			// punctuation characters.
-			continue
+			tor, err = torrentService.GetTorrentsOrderByNoCount(&parameters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
 		}
 
-		// TEMP: Workaround to at least make SQLite search testable for
-		// development.
-		// TODO: Actual case-insensitive search for SQLite
-		var operator string
-		if db.ORM.Dialect().GetName() == "sqlite3" {
-			operator = "LIKE ?"
-		} else {
-			operator = "ILIKE ?"
-		}
+		return
+	})
 
-		// TODO: make this faster ?
-		conditions = append(conditions, "torrent_name "+operator)
-		parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%")
-	}
-
-	parameters.Conditions = strings.Join(conditions[:], " AND ")
-	log.Infof("SQL query is :: %s\n", parameters.Conditions)
-	if countAll {
-		tor, count, err = torrentService.GetTorrentsOrderBy(&parameters, orderBy, int(search.Max), int(search.Max)*(pagenum-1))
-	} else {
-		tor, err = torrentService.GetTorrentsOrderByNoCount(&parameters, orderBy, int(search.Max), int(search.Max)*(pagenum-1))
-	}
 	return
 }