Merge branch 'cache-interface' into merge-cache-interface

cache/cache.go

@@ -0,0 +1,37 @@
+package cache
+
+import (
+	"github.com/ewhal/nyaa/cache/memcache"
+	"github.com/ewhal/nyaa/cache/native"
+	"github.com/ewhal/nyaa/cache/nop"
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/config"
+	"github.com/ewhal/nyaa/model"
+
+	"errors"
+)
+
+// Cache defines interface for caching search results
+type Cache interface {
+	Get(key common.SearchParam, r func() ([]model.Torrent, int, error)) ([]model.Torrent, int, error)
+	ClearAll()
+}
+
+var ErrInvalidCacheDialect = errors.New("invalid cache dialect")
+
+// Impl cache implementation instance
+var Impl Cache
+
+func Configure(conf *config.CacheConfig) (err error) {
+	switch conf.Dialect {
+	case "native":
+		Impl = native.New(conf.Size)
+		return
+	case "memcache":
+		Impl = memcache.New()
+		return
+	default:
+		Impl = nop.New()
+	}
+	return
+}

cache/memcache/memcache.go

@@ -0,0 +1,21 @@
+package memcache
+
+import (
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/model"
+)
+
+type Memcache struct {
+}
+
+func (c *Memcache) Get(key common.SearchParam, r func() ([]model.Torrent, int, error)) (torrents []model.Torrent, num int, err error) {
+	return
+}
+
+func (c *Memcache) ClearAll() {
+
+}
+
+func New() *Memcache {
+	return &Memcache{}
+}

cache/native/native.go

@@ -0,0 +1,143 @@
+package native
+
+import (
+	"container/list"
+	"sync"
+	"time"
+
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/model"
+)
+
+const expiryTime = time.Minute
+
+// NativeCache implements cache.Cache
+type NativeCache struct {
+	cache     map[common.SearchParam]*list.Element
+	ll        *list.List
+	totalUsed int
+	mu        sync.Mutex
+
+	// Size sets the maximum size of the cache before evicting unread data in MB
+	Size float64
+}
+
+// New Creates New Native Cache instance
+func New(sz float64) *NativeCache {
+	return &NativeCache{
+		cache: make(map[common.SearchParam]*list.Element, 10),
+		Size:  sz,
+		ll:    list.New(),
+	}
+}
+
+// Key stores the ID of either a thread or board page
+type Key struct {
+	LastN uint8
+	Board string
+	ID    uint64
+}
+
+// Single cache entry
+type store struct {
+	sync.Mutex  // Controls general access to the contents of the struct
+	lastFetched time.Time
+	key         common.SearchParam
+	data        []model.Torrent
+	count, size int
+	n           *NativeCache
+}
+
+// Check the cache for and existing record. If miss, run fn to retrieve fresh
+// values.
+func (n *NativeCache) Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) (
+	data []model.Torrent, count int, err error,
+) {
+	s := n.getStore(key)
+
+	// Also keeps multiple requesters from simultaneously requesting the same
+	// data
+	s.Lock()
+	defer s.Unlock()
+
+	if s.isFresh() {
+		return s.data, s.count, nil
+	}
+
+	data, count, err = fn()
+	if err != nil {
+		return
+	}
+	s.update(data, count)
+	return
+}
+
+// Retrieve a store from the cache or create a new one
+func (n *NativeCache) getStore(k common.SearchParam) (s *store) {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	el := n.cache[k]
+	if el == nil {
+		s = &store{key: k, n: n}
+		n.cache[k] = n.ll.PushFront(s)
+	} else {
+		n.ll.MoveToFront(el)
+		s = el.Value.(*store)
+	}
+	return s
+}
+
+// Clear the cache. Only used for testing.
+func (n *NativeCache) ClearAll() {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	n.ll = list.New()
+	n.cache = make(map[common.SearchParam]*list.Element, 10)
+}
+
+// Update the total used memory counter and evict, if over limit
+func (n *NativeCache) updateUsedSize(delta int) {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	n.totalUsed += delta
+
+	for n.totalUsed > int(n.Size)<<20 {
+		e := n.ll.Back()
+		if e == nil {
+			break
+		}
+		s := n.ll.Remove(e).(*store)
+		delete(n.cache, s.key)
+		n.totalUsed -= s.size
+	}
+}
+
+// Return, if the data can still be considered fresh, without querying the DB
+func (s *store) isFresh() bool {
+	if s.lastFetched.IsZero() { // New store
+		return false
+	}
+	return s.lastFetched.Add(expiryTime).After(time.Now())
+}
+
+// Stores the new values of s. Calculates and stores the new size. Passes the
+// delta to the central cache to fire eviction checks.
+func (s *store) update(data []model.Torrent, count int) {
+	newSize := 0
+	for _, d := range data {
+		newSize += d.Size()
+	}
+	s.data = data
+	s.count = count
+	delta := newSize - s.size
+	s.size = newSize
+	s.lastFetched = time.Now()
+
+	// Technically it is possible to update the size even when the store is
+	// already evicted, but that should never happen, unless you have a very
+	// small cache, very large stored datasets and a lot of traffic.
+	s.n.updateUsedSize(delta)
+}

cache/nop/nop.go

@@ -0,0 +1,22 @@
+package nop
+
+import (
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/model"
+)
+
+type NopCache struct {
+}
+
+func (c *NopCache) Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) ([]model.Torrent, int, error) {
+	return fn()
+}
+
+func (c *NopCache) ClearAll() {
+
+}
+
+// New creates a new Cache that does NOTHING :D
+func New() *NopCache {
+	return &NopCache{}
+}

config/cache.go

@@ -0,0 +1,14 @@
+package config
+
+// CacheConfig is config struct for caching strategy
+type CacheConfig struct {
+	Dialect string
+	URL     string
+	Size    float64
+}
+
+const DefaultCacheSize = 1 << 10
+
+var DefaultCacheConfig = CacheConfig{
+	Dialect: "nop",
+}

config/config.go

@@ -24,11 +24,15 @@ type Config struct {
 	DBParams string `json:"db_params"`
 	// tracker scraper config (required)
 	Scrape ScraperConfig `json:"scraper"`
+	// cache config
+	Cache CacheConfig `json:"cache"`
+	// search config
+	Search SearchConfig `json:"search"`
 	// optional i2p configuration
 	I2P *I2PConfig `json:"i2p"`
 }
 
-var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", DefaultScraperConfig, nil}
+var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", DefaultScraperConfig, DefaultCacheConfig, DefaultSearchConfig, nil}
 
 var allowedDatabaseTypes = map[string]bool{
 	"sqlite3":  true,
@@ -44,6 +48,7 @@ func New() *Config {
 	config.DBType = Defaults.DBType
 	config.DBParams = Defaults.DBParams
 	config.Scrape = Defaults.Scrape
+	config.Cache = Defaults.Cache
 	return &config
 }
 

main.go

@@ -9,12 +9,14 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/ewhal/nyaa/cache"
 	"github.com/ewhal/nyaa/config"
 	"github.com/ewhal/nyaa/db"
 	"github.com/ewhal/nyaa/network"
 	"github.com/ewhal/nyaa/router"
 	"github.com/ewhal/nyaa/service/scraper"
 	"github.com/ewhal/nyaa/util/log"
+	"github.com/ewhal/nyaa/util/search"
 	"github.com/ewhal/nyaa/util/signals"
 	"github.com/nicksnyder/go-i18n/i18n"
 )
@@ -97,6 +99,8 @@ func main() {
 	processFlags := conf.BindFlags()
 	defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout")
 	mode := flag.String("mode", "webapp", "which mode to run daemon in, either webapp or scraper")
+	flag.Float64Var(&conf.Cache.Size, "c", config.DefaultCacheSize, "size of the search cache in MB")
+
 	flag.Parse()
 	if *defaults {
 		stdout := bufio.NewWriter(os.Stdout)
@@ -119,6 +123,14 @@ func main() {
 			log.Fatal(err.Error())
 		}
 		initI18N()
+		err = cache.Configure(&conf.Cache)
+		if err != nil {
+			log.Fatal(err.Error())
+		}
+		err = search.Configure(&conf.Search)
+		if err != nil {
+			log.Fatal(err.Error())
+		}
 		go signals.Handle()
 		if len(config.TorrentFileStorage) > 0 {
 			err := os.MkdirAll(config.TorrentFileStorage, 0700)

router/homeHandler.go

@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"strconv"
 
+	"github.com/ewhal/nyaa/cache"
+	"github.com/ewhal/nyaa/common"
 	"github.com/ewhal/nyaa/model"
 	"github.com/ewhal/nyaa/service/torrent"
 	"github.com/ewhal/nyaa/util"
@@ -37,11 +39,19 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1))
+	search := common.SearchParam{
-	if !log.CheckError(err) {
+		Max:  uint(maxPerPage),
-		util.SendError(w, err, 400)
+		Page: pagenum,
 	}
 
+	torrents, nbTorrents, err := cache.Impl.Get(search, func() ([]model.Torrent, int, error) {
+		torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1))
+		if !log.CheckError(err) {
+			util.SendError(w, err, 400)
+		}
+		return torrents, nbTorrents, err
+	})
+
 	b := model.TorrentsToJSON(torrents)
 
 	navigationTorrents := Navigation{nbTorrents, maxPerPage, pagenum, "search_page"}

router/upload.go

@@ -14,6 +14,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/ewhal/nyaa/cache"
 	"github.com/ewhal/nyaa/config"
 	"github.com/ewhal/nyaa/service/captcha"
 	"github.com/ewhal/nyaa/util"
@@ -92,6 +93,7 @@ func (f *UploadForm) ExtractInfo(r *http.Request) error {
 	f.Name = util.TrimWhitespaces(f.Name)
 	f.Description = p.Sanitize(util.TrimWhitespaces(f.Description))
 	f.Magnet = util.TrimWhitespaces(f.Magnet)
+	cache.Impl.ClearAll()
 
 	catsSplit := strings.Split(f.Category, "_")
 	// need this to prevent out of index panics

util/search/search.go

@@ -7,7 +7,9 @@ import (
 	"unicode"
 	"unicode/utf8"
 
+	"github.com/ewhal/nyaa/cache"
 	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/config"
 	"github.com/ewhal/nyaa/db"
 	"github.com/ewhal/nyaa/model"
 	"github.com/ewhal/nyaa/service"
@@ -15,6 +17,18 @@ import (
 	"github.com/ewhal/nyaa/util/log"
 )
 
+var searchOperator string
+
+func Configure(conf *config.SearchConfig) (err error) {
+	// SQLite has case-insensitive LIKE, but no ILIKE
+	if db.ORM.Dialect().GetName() == "sqlite3" {
+		searchOperator = "LIKE ?"
+	} else {
+		searchOperator = "ILIKE ?"
+	}
+	return
+}
+
 func SearchByQuery(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, count int, err error) {
 	search, tor, count, err = searchByQuery(r, pagenum, true)
 	return
@@ -116,11 +130,11 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (
 	default:
 		orderBy += "desc"
 	}
-
 	parameters := serviceBase.WhereParams{
 		Params: make([]interface{}, 0, 64),
 	}
 	conditions := make([]string, 0, 64)
+
 	if search.Category.Main != 0 {
 		conditions = append(conditions, "category = ?")
 		parameters.Params = append(parameters.Params, string(catString[0]))
@@ -144,6 +158,11 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (
 	if len(search.NotNull) > 0 {
 		conditions = append(conditions, search.NotNull)
 	}
+
+	if len(search.NotNull) > 0 {
+		conditions = append(conditions, search.NotNull)
+	}
+
 	searchQuerySplit := strings.Fields(search.Query)
 	for i, word := range searchQuerySplit {
 		firstRune, _ := utf8.DecodeRuneInString(word)
@@ -156,26 +175,23 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (
 			continue
 		}
 
-		// SQLite has case-insensitive LIKE, but no ILIKE
-		var operator string
-		if db.ORM.Dialect().GetName() == "sqlite3" {
-			operator = "LIKE ?"
-		} else {
-			operator = "ILIKE ?"
-		}
-
 		// TODO: make this faster ?
-		conditions = append(conditions, "torrent_name "+operator)
+		conditions = append(conditions, "torrent_name "+searchOperator)
 		parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%")
 	}
 
 	parameters.Conditions = strings.Join(conditions[:], " AND ")
-	log.Infof("SQL query is :: %s\n", parameters.Conditions)
-	if countAll {
-		tor, count, err = torrentService.GetTorrentsOrderBy(&parameters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
-	} else {
-		tor, err = torrentService.GetTorrentsOrderByNoCount(&parameters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
-	}
 
+	log.Infof("SQL query is :: %s\n", parameters.Conditions)
+
+	tor, count, err = cache.Impl.Get(search, func() (tor []model.Torrent, count int, err error) {
+
+		if countAll {
+			tor, count, err = torrentService.GetTorrentsOrderBy(&parameters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
+		} else {
+			tor, err = torrentService.GetTorrentsOrderByNoCount(&parameters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
+		}
+		return
+	})
 	return
 }