Remove broken caching stuff
Cette révision appartient à :
Eliot Whalan
Parent
a17004360f
révision
c1cafb9d9a
5 fichiers modifiés avec 60 ajouts et 210 suppressions
132
cache/cache.go
externe
132
cache/cache.go
externe
|
|
@ -1,132 +0,0 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ewhal/nyaa/common"
|
||||
"github.com/ewhal/nyaa/model"
|
||||
)
|
||||
|
||||
const expiryTime = time.Minute
|
||||
|
||||
var (
|
||||
cache = make(map[common.SearchParam]*list.Element, 10)
|
||||
ll = list.New()
|
||||
totalUsed int
|
||||
mu sync.Mutex
|
||||
|
||||
// Size sets the maximum size of the cache before evicting unread data in MB
|
||||
Size float64 = 1 << 10
|
||||
)
|
||||
|
||||
// Key stores the ID of either a thread or board page
|
||||
type Key struct {
|
||||
LastN uint8
|
||||
Board string
|
||||
ID uint64
|
||||
}
|
||||
|
||||
// Single cache entry
|
||||
type store struct {
|
||||
sync.Mutex // Controls general access to the contents of the struct
|
||||
lastFetched time.Time
|
||||
key common.SearchParam
|
||||
data []model.Torrent
|
||||
count, size int
|
||||
}
|
||||
|
||||
// Check the cache for and existing record. If miss, run fn to retrieve fresh
|
||||
// values.
|
||||
func Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) (
|
||||
data []model.Torrent, count int, err error,
|
||||
) {
|
||||
s := getStore(key)
|
||||
|
||||
// Also keeps multiple requesters from simultaneously requesting the same
|
||||
// data
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
if s.isFresh() {
|
||||
return s.data, s.count, nil
|
||||
}
|
||||
|
||||
data, count, err = fn()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.update(data, count)
|
||||
return
|
||||
}
|
||||
|
||||
// Retrieve a store from the cache or create a new one
|
||||
func getStore(k common.SearchParam) (s *store) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
el := cache[k]
|
||||
if el == nil {
|
||||
s = &store{key: k}
|
||||
cache[k] = ll.PushFront(s)
|
||||
} else {
|
||||
ll.MoveToFront(el)
|
||||
s = el.Value.(*store)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Clear the cache. Only used for testing.
|
||||
func Clear() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
ll = list.New()
|
||||
cache = make(map[common.SearchParam]*list.Element, 10)
|
||||
}
|
||||
|
||||
// Update the total used memory counter and evict, if over limit
|
||||
func updateUsedSize(delta int) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
totalUsed += delta
|
||||
|
||||
for totalUsed > int(Size)<<20 {
|
||||
e := ll.Back()
|
||||
if e == nil {
|
||||
break
|
||||
}
|
||||
s := ll.Remove(e).(*store)
|
||||
delete(cache, s.key)
|
||||
totalUsed -= s.size
|
||||
}
|
||||
}
|
||||
|
||||
// Return, if the data can still be considered fresh, without querying the DB
|
||||
func (s *store) isFresh() bool {
|
||||
if s.lastFetched.IsZero() { // New store
|
||||
return false
|
||||
}
|
||||
return s.lastFetched.Add(expiryTime).After(time.Now())
|
||||
}
|
||||
|
||||
// Stores the new values of s. Calculates and stores the new size. Passes the
|
||||
// delta to the central cache to fire eviction checks.
|
||||
func (s *store) update(data []model.Torrent, count int) {
|
||||
newSize := 0
|
||||
for _, d := range data {
|
||||
newSize += d.Size()
|
||||
}
|
||||
s.data = data
|
||||
s.count = count
|
||||
delta := newSize - s.size
|
||||
s.size = newSize
|
||||
s.lastFetched = time.Now()
|
||||
|
||||
// Technically it is possible to update the size even when the store is
|
||||
// already evicted, but that should never happen, unless you have a very
|
||||
// small cache, very large stored datasets and a lot of traffic.
|
||||
updateUsedSize(delta)
|
||||
}
|
||||
2
main.go
2
main.go
|
|
@ -9,7 +9,6 @@ import (
|
|||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/ewhal/nyaa/cache"
|
||||
"github.com/ewhal/nyaa/config"
|
||||
"github.com/ewhal/nyaa/db"
|
||||
"github.com/ewhal/nyaa/network"
|
||||
|
|
@ -98,7 +97,6 @@ func main() {
|
|||
processFlags := conf.BindFlags()
|
||||
defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout")
|
||||
mode := flag.String("mode", "webapp", "which mode to run daemon in, either webapp or scraper")
|
||||
flag.Float64Var(&cache.Size, "c", cache.Size, "size of the search cache in MB")
|
||||
flag.Parse()
|
||||
if *defaults {
|
||||
stdout := bufio.NewWriter(os.Stdout)
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@ import (
|
|||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/ewhal/nyaa/cache"
|
||||
"github.com/ewhal/nyaa/common"
|
||||
"github.com/ewhal/nyaa/model"
|
||||
"github.com/ewhal/nyaa/service/torrent"
|
||||
"github.com/ewhal/nyaa/util"
|
||||
|
|
@ -39,17 +37,10 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
}
|
||||
|
||||
search := common.SearchParam{
|
||||
Max: uint(maxPerPage),
|
||||
Page: pagenum,
|
||||
torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1))
|
||||
if !log.CheckError(err) {
|
||||
util.SendError(w, err, 400)
|
||||
}
|
||||
torrents, nbTorrents, err := cache.Get(search, func() ([]model.Torrent, int, error) {
|
||||
torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1))
|
||||
if !log.CheckError(err) {
|
||||
util.SendError(w, err, 400)
|
||||
}
|
||||
return torrents, nbTorrents, err
|
||||
})
|
||||
|
||||
b := model.TorrentsToJSON(torrents)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/ewhal/nyaa/cache"
|
||||
"github.com/ewhal/nyaa/config"
|
||||
"github.com/ewhal/nyaa/service/captcha"
|
||||
"github.com/ewhal/nyaa/util"
|
||||
|
|
@ -30,7 +29,7 @@ type UploadForm struct {
|
|||
Category string
|
||||
Remake bool
|
||||
Description string
|
||||
Status int
|
||||
Status int
|
||||
captcha.Captcha
|
||||
|
||||
Infohash string
|
||||
|
|
@ -93,7 +92,6 @@ func (f *UploadForm) ExtractInfo(r *http.Request) error {
|
|||
f.Name = util.TrimWhitespaces(f.Name)
|
||||
f.Description = p.Sanitize(util.TrimWhitespaces(f.Description))
|
||||
f.Magnet = util.TrimWhitespaces(f.Magnet)
|
||||
cache.Clear()
|
||||
|
||||
catsSplit := strings.Split(f.Category, "_")
|
||||
// need this to prevent out of index panics
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import (
|
|||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/ewhal/nyaa/cache"
|
||||
"github.com/ewhal/nyaa/common"
|
||||
"github.com/ewhal/nyaa/db"
|
||||
"github.com/ewhal/nyaa/model"
|
||||
|
|
@ -118,69 +117,65 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (
|
|||
orderBy += "desc"
|
||||
}
|
||||
|
||||
tor, count, err = cache.Get(search, func() (tor []model.Torrent, count int, err error) {
|
||||
parameters := serviceBase.WhereParams{
|
||||
Params: make([]interface{}, 0, 64),
|
||||
}
|
||||
conditions := make([]string, 0, 64)
|
||||
if search.Category.Main != 0 {
|
||||
conditions = append(conditions, "category = ?")
|
||||
parameters.Params = append(parameters.Params, string(catString[0]))
|
||||
}
|
||||
if search.UserID != 0 {
|
||||
conditions = append(conditions, "uploader = ?")
|
||||
parameters.Params = append(parameters.Params, search.UserID)
|
||||
}
|
||||
if search.Category.Sub != 0 {
|
||||
conditions = append(conditions, "sub_category = ?")
|
||||
parameters.Params = append(parameters.Params, string(catString[2]))
|
||||
}
|
||||
if search.Status != 0 {
|
||||
if search.Status == common.FilterRemakes {
|
||||
conditions = append(conditions, "status > ?")
|
||||
} else {
|
||||
conditions = append(conditions, "status >= ?")
|
||||
}
|
||||
parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1))
|
||||
}
|
||||
if len(search.NotNull) > 0 {
|
||||
conditions = append(conditions, search.NotNull)
|
||||
}
|
||||
searchQuerySplit := strings.Fields(search.Query)
|
||||
for i, word := range searchQuerySplit {
|
||||
firstRune, _ := utf8.DecodeRuneInString(word)
|
||||
if len(word) == 1 && unicode.IsPunct(firstRune) {
|
||||
// some queries have a single punctuation character
|
||||
// which causes a full scan instead of using the index
|
||||
// and yields no meaningful results.
|
||||
// due to len() == 1 we're just looking at 1-byte/ascii
|
||||
// punctuation characters.
|
||||
continue
|
||||
}
|
||||
|
||||
// SQLite has case-insensitive LIKE, but no ILIKE
|
||||
var operator string
|
||||
if db.ORM.Dialect().GetName() == "sqlite3" {
|
||||
operator = "LIKE ?"
|
||||
} else {
|
||||
operator = "ILIKE ?"
|
||||
}
|
||||
|
||||
// TODO: make this faster ?
|
||||
conditions = append(conditions, "torrent_name "+operator)
|
||||
parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%")
|
||||
}
|
||||
|
||||
parameters.Conditions = strings.Join(conditions[:], " AND ")
|
||||
log.Infof("SQL query is :: %s\n", parameters.Conditions)
|
||||
if countAll {
|
||||
tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
|
||||
parameters := serviceBase.WhereParams{
|
||||
Params: make([]interface{}, 0, 64),
|
||||
}
|
||||
conditions := make([]string, 0, 64)
|
||||
if search.Category.Main != 0 {
|
||||
conditions = append(conditions, "category = ?")
|
||||
parameters.Params = append(parameters.Params, string(catString[0]))
|
||||
}
|
||||
if search.UserID != 0 {
|
||||
conditions = append(conditions, "uploader = ?")
|
||||
parameters.Params = append(parameters.Params, search.UserID)
|
||||
}
|
||||
if search.Category.Sub != 0 {
|
||||
conditions = append(conditions, "sub_category = ?")
|
||||
parameters.Params = append(parameters.Params, string(catString[2]))
|
||||
}
|
||||
if search.Status != 0 {
|
||||
if search.Status == common.FilterRemakes {
|
||||
conditions = append(conditions, "status > ?")
|
||||
} else {
|
||||
tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
|
||||
conditions = append(conditions, "status >= ?")
|
||||
}
|
||||
parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1))
|
||||
}
|
||||
if len(search.NotNull) > 0 {
|
||||
conditions = append(conditions, search.NotNull)
|
||||
}
|
||||
searchQuerySplit := strings.Fields(search.Query)
|
||||
for i, word := range searchQuerySplit {
|
||||
firstRune, _ := utf8.DecodeRuneInString(word)
|
||||
if len(word) == 1 && unicode.IsPunct(firstRune) {
|
||||
// some queries have a single punctuation character
|
||||
// which causes a full scan instead of using the index
|
||||
// and yields no meaningful results.
|
||||
// due to len() == 1 we're just looking at 1-byte/ascii
|
||||
// punctuation characters.
|
||||
continue
|
||||
}
|
||||
|
||||
return
|
||||
})
|
||||
// SQLite has case-insensitive LIKE, but no ILIKE
|
||||
var operator string
|
||||
if db.ORM.Dialect().GetName() == "sqlite3" {
|
||||
operator = "LIKE ?"
|
||||
} else {
|
||||
operator = "ILIKE ?"
|
||||
}
|
||||
|
||||
// TODO: make this faster ?
|
||||
conditions = append(conditions, "torrent_name "+operator)
|
||||
parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%")
|
||||
}
|
||||
|
||||
parameters.Conditions = strings.Join(conditions[:], " AND ")
|
||||
log.Infof("SQL query is :: %s\n", parameters.Conditions)
|
||||
if countAll {
|
||||
tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
|
||||
} else {
|
||||
tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1))
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
|
|
|||
Référencer dans un nouveau ticket