diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 00000000..98eab0de --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,37 @@ +package cache + +import ( + "github.com/ewhal/nyaa/cache/memcache" + "github.com/ewhal/nyaa/cache/native" + "github.com/ewhal/nyaa/cache/nop" + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/config" + "github.com/ewhal/nyaa/model" + + "errors" +) + +// Cache defines interface for caching search results +type Cache interface { + Get(key common.SearchParam, r func() ([]model.Torrent, int, error)) ([]model.Torrent, int, error) + ClearAll() +} + +var ErrInvalidCacheDialect = errors.New("invalid cache dialect") + +// Impl cache implementation instance +var Impl Cache + +func Configure(conf *config.CacheConfig) (err error) { + switch conf.Dialect { + case "native": + Impl = native.New(conf.Size) + return + case "memcache": + Impl = memcache.New() + return + default: + Impl = nop.New() + } + return +} diff --git a/cache/memcache/memcache.go b/cache/memcache/memcache.go new file mode 100644 index 00000000..57f36d96 --- /dev/null +++ b/cache/memcache/memcache.go @@ -0,0 +1,21 @@ +package memcache + +import ( + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/model" +) + +type Memcache struct { +} + +func (c *Memcache) Get(key common.SearchParam, r func() ([]model.Torrent, int, error)) (torrents []model.Torrent, num int, err error) { + return +} + +func (c *Memcache) ClearAll() { + +} + +func New() *Memcache { + return &Memcache{} +} diff --git a/cache/native/native.go b/cache/native/native.go new file mode 100644 index 00000000..c9b3ba36 --- /dev/null +++ b/cache/native/native.go @@ -0,0 +1,143 @@ +package native + +import ( + "container/list" + "sync" + "time" + + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/model" +) + +const expiryTime = time.Minute + +// NativeCache implements cache.Cache +type NativeCache struct { + cache map[common.SearchParam]*list.Element + ll *list.List + totalUsed int + mu sync.Mutex + + // Size sets the maximum size of the cache before evicting unread data in MB + Size float64 +} + +// New Creates New Native Cache instance +func New(sz float64) *NativeCache { + return &NativeCache{ + cache: make(map[common.SearchParam]*list.Element, 10), + Size: sz, + ll: list.New(), + } +} + +// Key stores the ID of either a thread or board page +type Key struct { + LastN uint8 + Board string + ID uint64 +} + +// Single cache entry +type store struct { + sync.Mutex // Controls general access to the contents of the struct + lastFetched time.Time + key common.SearchParam + data []model.Torrent + count, size int + n *NativeCache +} + +// Check the cache for and existing record. If miss, run fn to retrieve fresh +// values. +func (n *NativeCache) Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) ( + data []model.Torrent, count int, err error, +) { + s := n.getStore(key) + + // Also keeps multiple requesters from simultaneously requesting the same + // data + s.Lock() + defer s.Unlock() + + if s.isFresh() { + return s.data, s.count, nil + } + + data, count, err = fn() + if err != nil { + return + } + s.update(data, count) + return +} + +// Retrieve a store from the cache or create a new one +func (n *NativeCache) getStore(k common.SearchParam) (s *store) { + n.mu.Lock() + defer n.mu.Unlock() + + el := n.cache[k] + if el == nil { + s = &store{key: k, n: n} + n.cache[k] = n.ll.PushFront(s) + } else { + n.ll.MoveToFront(el) + s = el.Value.(*store) + } + return s +} + +// Clear the cache. Only used for testing. +func (n *NativeCache) ClearAll() { + n.mu.Lock() + defer n.mu.Unlock() + + n.ll = list.New() + n.cache = make(map[common.SearchParam]*list.Element, 10) +} + +// Update the total used memory counter and evict, if over limit +func (n *NativeCache) updateUsedSize(delta int) { + n.mu.Lock() + defer n.mu.Unlock() + + n.totalUsed += delta + + for n.totalUsed > int(n.Size)<<20 { + e := n.ll.Back() + if e == nil { + break + } + s := n.ll.Remove(e).(*store) + delete(n.cache, s.key) + n.totalUsed -= s.size + } +} + +// Return, if the data can still be considered fresh, without querying the DB +func (s *store) isFresh() bool { + if s.lastFetched.IsZero() { // New store + return false + } + return s.lastFetched.Add(expiryTime).After(time.Now()) +} + +// Stores the new values of s. Calculates and stores the new size. Passes the +// delta to the central cache to fire eviction checks. +func (s *store) update(data []model.Torrent, count int) { + newSize := 0 + for _, d := range data { + newSize += d.Size() + } + s.data = data + s.count = count + delta := newSize - s.size + s.size = newSize + s.lastFetched = time.Now() + + // Technically it is possible to update the size even when the store is + // already evicted, but that should never happen, unless you have a very + // small cache, very large stored datasets and a lot of traffic. + s.n.updateUsedSize(delta) +} diff --git a/cache/nop/nop.go b/cache/nop/nop.go new file mode 100644 index 00000000..21c10580 --- /dev/null +++ b/cache/nop/nop.go @@ -0,0 +1,22 @@ +package nop + +import ( + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/model" +) + +type NopCache struct { +} + +func (c *NopCache) Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) ([]model.Torrent, int, error) { + return fn() +} + +func (c *NopCache) ClearAll() { + +} + +// New creates a new Cache that does NOTHING :D +func New() *NopCache { + return &NopCache{} +} diff --git a/config/cache.go b/config/cache.go new file mode 100644 index 00000000..93e195fb --- /dev/null +++ b/config/cache.go @@ -0,0 +1,14 @@ +package config + +// CacheConfig is config struct for caching strategy +type CacheConfig struct { + Dialect string + URL string + Size float64 +} + +const DefaultCacheSize = 1 << 10 + +var DefaultCacheConfig = CacheConfig{ + Dialect: "nop", +} diff --git a/config/config.go b/config/config.go index bea5b8d9..ae599c23 100644 --- a/config/config.go +++ b/config/config.go @@ -24,11 +24,15 @@ type Config struct { DBParams string `json:"db_params"` // tracker scraper config (required) Scrape ScraperConfig `json:"scraper"` + // cache config + Cache CacheConfig `json:"cache"` + // search config + Search SearchConfig `json:"search"` // optional i2p configuration I2P *I2PConfig `json:"i2p"` } -var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", DefaultScraperConfig, nil} +var Defaults = Config{"localhost", 9999, "sqlite3", "./nyaa.db?cache_size=50", DefaultScraperConfig, DefaultCacheConfig, DefaultSearchConfig, nil} var allowedDatabaseTypes = map[string]bool{ "sqlite3": true, @@ -44,6 +48,7 @@ func New() *Config { config.DBType = Defaults.DBType config.DBParams = Defaults.DBParams config.Scrape = Defaults.Scrape + config.Cache = Defaults.Cache return &config } diff --git a/main.go b/main.go index 700b7a47..07260f53 100644 --- a/main.go +++ b/main.go @@ -9,12 +9,14 @@ import ( "path/filepath" "time" + "github.com/ewhal/nyaa/cache" "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/network" "github.com/ewhal/nyaa/router" "github.com/ewhal/nyaa/service/scraper" "github.com/ewhal/nyaa/util/log" + "github.com/ewhal/nyaa/util/search" "github.com/ewhal/nyaa/util/signals" "github.com/nicksnyder/go-i18n/i18n" ) @@ -97,6 +99,8 @@ func main() { processFlags := conf.BindFlags() defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout") mode := flag.String("mode", "webapp", "which mode to run daemon in, either webapp or scraper") + flag.Float64Var(&conf.Cache.Size, "c", config.DefaultCacheSize, "size of the search cache in MB") + flag.Parse() if *defaults { stdout := bufio.NewWriter(os.Stdout) @@ -119,6 +123,14 @@ func main() { log.Fatal(err.Error()) } initI18N() + err = cache.Configure(&conf.Cache) + if err != nil { + log.Fatal(err.Error()) + } + err = search.Configure(&conf.Search) + if err != nil { + log.Fatal(err.Error()) + } go signals.Handle() if len(config.TorrentFileStorage) > 0 { err := os.MkdirAll(config.TorrentFileStorage, 0700) diff --git a/router/homeHandler.go b/router/homeHandler.go index 0735515b..51cd5c06 100644 --- a/router/homeHandler.go +++ b/router/homeHandler.go @@ -5,6 +5,8 @@ import ( "net/http" "strconv" + "github.com/ewhal/nyaa/cache" + "github.com/ewhal/nyaa/common" "github.com/ewhal/nyaa/model" "github.com/ewhal/nyaa/service/torrent" "github.com/ewhal/nyaa/util" @@ -37,11 +39,19 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) { } } - torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) - if !log.CheckError(err) { - util.SendError(w, err, 400) + search := common.SearchParam{ + Max: uint(maxPerPage), + Page: pagenum, } + torrents, nbTorrents, err := cache.Impl.Get(search, func() ([]model.Torrent, int, error) { + torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) + if !log.CheckError(err) { + util.SendError(w, err, 400) + } + return torrents, nbTorrents, err + }) + b := model.TorrentsToJSON(torrents) navigationTorrents := Navigation{nbTorrents, maxPerPage, pagenum, "search_page"} diff --git a/router/upload.go b/router/upload.go index 1419dc63..42664718 100644 --- a/router/upload.go +++ b/router/upload.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" + "github.com/ewhal/nyaa/cache" "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/service/captcha" "github.com/ewhal/nyaa/util" @@ -92,6 +93,7 @@ func (f *UploadForm) ExtractInfo(r *http.Request) error { f.Name = util.TrimWhitespaces(f.Name) f.Description = p.Sanitize(util.TrimWhitespaces(f.Description)) f.Magnet = util.TrimWhitespaces(f.Magnet) + cache.Impl.ClearAll() catsSplit := strings.Split(f.Category, "_") // need this to prevent out of index panics diff --git a/util/search/search.go b/util/search/search.go index a30a08ca..60472a8f 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -7,7 +7,9 @@ import ( "unicode" "unicode/utf8" + "github.com/ewhal/nyaa/cache" "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/model" "github.com/ewhal/nyaa/service" @@ -15,6 +17,18 @@ import ( "github.com/ewhal/nyaa/util/log" ) +var searchOperator string + +func Configure(conf *config.SearchConfig) (err error) { + // SQLite has case-insensitive LIKE, but no ILIKE + if db.ORM.Dialect().GetName() == "sqlite3" { + searchOperator = "LIKE ?" + } else { + searchOperator = "ILIKE ?" + } + return +} + func SearchByQuery(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, count int, err error) { search, tor, count, err = searchByQuery(r, pagenum, true) return @@ -116,11 +130,11 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( default: orderBy += "desc" } - parameters := serviceBase.WhereParams{ Params: make([]interface{}, 0, 64), } conditions := make([]string, 0, 64) + if search.Category.Main != 0 { conditions = append(conditions, "category = ?") parameters.Params = append(parameters.Params, string(catString[0])) @@ -144,6 +158,11 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( if len(search.NotNull) > 0 { conditions = append(conditions, search.NotNull) } + + if len(search.NotNull) > 0 { + conditions = append(conditions, search.NotNull) + } + searchQuerySplit := strings.Fields(search.Query) for i, word := range searchQuerySplit { firstRune, _ := utf8.DecodeRuneInString(word) @@ -156,26 +175,23 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) ( continue } - // SQLite has case-insensitive LIKE, but no ILIKE - var operator string - if db.ORM.Dialect().GetName() == "sqlite3" { - operator = "LIKE ?" - } else { - operator = "ILIKE ?" - } - // TODO: make this faster ? - conditions = append(conditions, "torrent_name "+operator) + conditions = append(conditions, "torrent_name "+searchOperator) parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") } parameters.Conditions = strings.Join(conditions[:], " AND ") - log.Infof("SQL query is :: %s\n", parameters.Conditions) - if countAll { - tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) - } else { - tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) - } + log.Infof("SQL query is :: %s\n", parameters.Conditions) + + tor, count, err = cache.Impl.Get(search, func() (tor []model.Torrent, count int, err error) { + + if countAll { + tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) + } else { + tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) + } + return + }) return }