diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 00000000..66c77c89 --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,128 @@ +package cache + +import ( + "container/list" + "sync" + "time" + + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/model" +) + +const expiryTime = time.Minute + +var ( + cache = make(map[common.SearchParam]*list.Element, 10) + ll = list.New() + totalUsed int + mu sync.Mutex + + // Size sets the maximum size of the cache before evicting unread data in MB + Size float64 = 1 << 10 +) + +// Key stores the ID of either a thread or board page +type Key struct { + LastN uint8 + Board string + ID uint64 +} + +// Single cache entry +type store struct { + sync.Mutex // Controls general access to the contents of the struct + lastFetched time.Time + key common.SearchParam + data []model.Torrent + count, size int +} + +// Check the cache for and existing record. If miss, run fn to retrieve fresh +// values. +func Get(key common.SearchParam, fn func() ([]model.Torrent, int, error)) ( + data []model.Torrent, count int, err error, +) { + s := getStore(key) + + // Also keeps multiple requesters from simultaneously requesting the same + // data + s.Lock() + defer s.Unlock() + + if s.isFresh() { + return s.data, s.count, nil + } + + data, count, err = fn() + if err != nil { + return + } + s.update(data, count) + return +} + +// Retrieve a store from the cache or create a new one +func getStore(k common.SearchParam) (s *store) { + mu.Lock() + defer mu.Unlock() + + el := cache[k] + if el == nil { + s = &store{key: k} + cache[k] = ll.PushFront(s) + } else { + ll.MoveToFront(el) + s = el.Value.(*store) + } + return s +} + +// Clear the cache. Only used for testing. +func Clear() { + mu.Lock() + defer mu.Unlock() + + ll = list.New() + cache = make(map[common.SearchParam]*list.Element, 10) +} + +// Update the total used memory counter and evict, if over limit +func updateUsedSize(delta int) { + mu.Lock() + defer mu.Unlock() + + totalUsed += delta + + for totalUsed > int(Size)<<20 { + s := ll.Remove(ll.Back()).(*store) + delete(cache, s.key) + totalUsed -= s.size + } +} + +// Return, if the data can still be considered fresh, without querying the DB +func (s *store) isFresh() bool { + if s.lastFetched.IsZero() { // New store + return false + } + return s.lastFetched.Add(expiryTime).After(time.Now()) +} + +// Stores the new values of s. Calculates and stores the new size. Passes the +// delta to the central cache to fire eviction checks. +func (s *store) update(data []model.Torrent, count int) { + newSize := 0 + for _, d := range data { + newSize += d.Size() + } + s.data = data + s.count = count + delta := newSize - s.size + s.size = newSize + s.lastFetched = time.Now() + + // Technically it is possible to update the size even when the store is + // already evicted, but that should never happen, unless you have a very + // small cache, very large stored datasets and a lot of traffic. + updateUsedSize(delta) +} diff --git a/common/search.go b/common/search.go new file mode 100644 index 00000000..92708073 --- /dev/null +++ b/common/search.go @@ -0,0 +1,47 @@ +package common + +import "strconv" + +type Status uint8 + +const ( + ShowAll Status = iota + FilterRemakes + Trusted + APlus +) + +type SortMode uint8 + +const ( + ID SortMode = iota + Name + Date + Downloads + Size +) + +type Category struct { + Main, Sub uint8 +} + +func (c Category) String() (s string) { + if c.Main != 0 { + s += strconv.Itoa(int(c.Main)) + } + s += "_" + if c.Sub != 0 { + s += strconv.Itoa(int(c.Sub)) + } + return +} + +type SearchParam struct { + Order bool // True means acsending + Status Status + Sort SortMode + Category Category + Page int + Max uint + Query string +} diff --git a/main.go b/main.go index dbec6dae..a0c8a3a4 100644 --- a/main.go +++ b/main.go @@ -3,20 +3,19 @@ package main import ( "bufio" "flag" + "net/http" + "os" + "path/filepath" + "time" - "github.com/nicksnyder/go-i18n/i18n" - + "github.com/ewhal/nyaa/cache" "github.com/ewhal/nyaa/config" "github.com/ewhal/nyaa/db" "github.com/ewhal/nyaa/network" "github.com/ewhal/nyaa/router" "github.com/ewhal/nyaa/util/log" "github.com/ewhal/nyaa/util/signals" - - "net/http" - "os" - "path/filepath" - "time" + "github.com/nicksnyder/go-i18n/i18n" ) func initI18N() { @@ -51,6 +50,7 @@ func main() { conf := config.New() processFlags := conf.BindFlags() defaults := flag.Bool("print-defaults", false, "print the default configuration file on stdout") + flag.Float64Var(&cache.Size, "c", cache.Size, "size of the search cache in MB") flag.Parse() if *defaults { stdout := bufio.NewWriter(os.Stdout) diff --git a/model/comment.go b/model/comment.go index 703d287f..d80c5398 100644 --- a/model/comment.go +++ b/model/comment.go @@ -17,6 +17,11 @@ type Comment struct { User *User `gorm:"ForeignKey:user_id"` } +// Returns the total size of memory recursively allocated for this struct +func (c Comment) Size() int { + return (3 + 3*3 + 2 + 2 + len(c.Content)) * 8 +} + type OldComment struct { TorrentID uint `gorm:"column:torrent_id"` Username string `gorm:"column:username"` @@ -26,6 +31,11 @@ type OldComment struct { Torrent *Torrent `gorm:"ForeignKey:torrent_id"` } +// Returns the total size of memory recursively allocated for this struct +func (c OldComment) Size() int { + return (1 + 2*2 + len(c.Username) + len(c.Content) + 3 + 1) * 8 +} + func (c OldComment) TableName() string { // cba to rename this in the db // TODO: Update database schema to fix this hack diff --git a/model/torrent.go b/model/torrent.go index 74fd488e..55eb2105 100644 --- a/model/torrent.go +++ b/model/torrent.go @@ -34,24 +34,49 @@ type Torrent struct { Filesize int64 `gorm:"column:filesize"` Description string `gorm:"column:description"` WebsiteLink string `gorm:"column:website_link"` - DeletedAt *time.Time + DeletedAt *time.Time Uploader *User `gorm:"ForeignKey:UploaderId"` OldComments []OldComment `gorm:"ForeignKey:torrent_id"` Comments []Comment `gorm:"ForeignKey:torrent_id"` } +// Returns the total size of memory recursively allocated for this struct +func (t Torrent) Size() (s int) { + s += 8 + // ints + 2*3 + // time.Time + 2 + // pointers + 4*2 + // string pointers + // string array sizes + len(t.Name) + len(t.Hash) + len(t.Description) + len(t.WebsiteLink) + + 2*2 // array pointers + s *= 8 // Assume 64 bit OS + + if t.Uploader != nil { + s += t.Uploader.Size() + } + for _, c := range t.OldComments { + s += c.Size() + } + for _, c := range t.Comments { + s += c.Size() + } + + return + +} + // TODO Add field to specify kind of reports // TODO Add CreatedAt field // INFO User can be null (anonymous reports) // FIXME can't preload field Torrents for model.TorrentReport type TorrentReport struct { - ID uint `gorm:"column:torrent_report_id;primary_key"` - Description string `gorm:"column:type"` - TorrentID uint - UserID uint - Torrent Torrent `gorm:"AssociationForeignKey:TorrentID;ForeignKey:ID"` - User User `gorm:"AssociationForeignKey:UserID;ForeignKey:ID"` + ID uint `gorm:"column:torrent_report_id;primary_key"` + Description string `gorm:"column:type"` + TorrentID uint + UserID uint + Torrent Torrent `gorm:"AssociationForeignKey:TorrentID;ForeignKey:ID"` + User User `gorm:"AssociationForeignKey:UserID;ForeignKey:ID"` } /* We need a JSON object instead of a Gorm structure because magnet URLs are @@ -89,9 +114,9 @@ type TorrentJSON struct { } type TorrentReportJson struct { - ID uint `json:"id"` - Description string `json:"description"` - Torrent TorrentJSON `json:"torrent"` + ID uint `json:"id"` + Description string `json:"description"` + Torrent TorrentJSON `json:"torrent"` User string } diff --git a/model/user.go b/model/user.go index 2b2595ca..4a01c168 100644 --- a/model/user.go +++ b/model/user.go @@ -26,6 +26,21 @@ type User struct { Torrents []Torrent `gorm:"ForeignKey:UploaderID"` } +// Returns the total size of memory recursively allocated for this struct +func (u User) Size() (s int) { + s += 4 + // ints + 6*2 + // string pointers + 4*3 + //time.Time + 3*2 + // arrays + // string arrays + len(u.Username) + len(u.Password) + len(u.Email) + len(u.Token) + len(u.MD5) + len(u.Language) + s *= 8 + + // Ignoring foreign key users. Fuck them. + + return +} + type PublicUser struct { User *User } diff --git a/router/homeHandler.go b/router/homeHandler.go index 55b16d90..ef2156e0 100644 --- a/router/homeHandler.go +++ b/router/homeHandler.go @@ -1,15 +1,18 @@ package router import ( + "html" + "net/http" + "strconv" + + "github.com/ewhal/nyaa/cache" + "github.com/ewhal/nyaa/common" "github.com/ewhal/nyaa/model" "github.com/ewhal/nyaa/service/torrent" "github.com/ewhal/nyaa/util" "github.com/ewhal/nyaa/util/languages" "github.com/ewhal/nyaa/util/log" "github.com/gorilla/mux" - "html" - "net/http" - "strconv" ) func HomeHandler(w http.ResponseWriter, r *http.Request) { @@ -36,11 +39,17 @@ func HomeHandler(w http.ResponseWriter, r *http.Request) { } } - torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) - if !log.CheckError(err) { - util.SendError(w, err, 400) - return + search := common.SearchParam{ + Max: uint(maxPerPage), + Page: pagenum, } + torrents, nbTorrents, err := cache.Get(search, func() ([]model.Torrent, int, error) { + torrents, nbTorrents, err := torrentService.GetAllTorrents(maxPerPage, maxPerPage*(pagenum-1)) + if !log.CheckError(err) { + util.SendError(w, err, 400) + } + return torrents, nbTorrents, err + }) b := model.TorrentsToJSON(torrents) diff --git a/router/templateVariables.go b/router/templateVariables.go index 32417d89..126b012d 100644 --- a/router/templateVariables.go +++ b/router/templateVariables.go @@ -4,11 +4,11 @@ import ( "net/http" "net/url" + "github.com/ewhal/nyaa/common" "github.com/ewhal/nyaa/model" "github.com/ewhal/nyaa/service/captcha" "github.com/ewhal/nyaa/service/user" userForms "github.com/ewhal/nyaa/service/user/form" - "github.com/ewhal/nyaa/util/search" "github.com/gorilla/mux" ) @@ -87,7 +87,7 @@ type UserLoginFormVariables struct { type UserProfileVariables struct { UserProfile *model.User - FormInfos map[string][]string + FormInfos map[string][]string Search SearchForm Navigation Navigation User *model.User @@ -114,22 +114,22 @@ type UploadTemplateVariables struct { } type PanelIndexVbs struct { - Torrents []model.Torrent - Users []model.User - Comments []model.Comment + Torrents []model.Torrent + Users []model.User + Comments []model.Comment } type PanelTorrentListVbs struct { - Torrents []model.Torrent + Torrents []model.Torrent } type PanelUserListVbs struct { - Users []model.User + Users []model.User } type PanelCommentListVbs struct { - Comments []model.Comment + Comments []model.Comment } type PanelTorrentEdVbs struct { - Torrent model.Torrent + Torrent model.Torrent } type ViewTorrentReportsVariables struct { @@ -147,7 +147,7 @@ type Navigation struct { } type SearchForm struct { - search.SearchParam + common.SearchParam Category string HideAdvancedSearch bool } diff --git a/util/search/search.go b/util/search/search.go index a1e35913..4f28bfdc 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -1,71 +1,33 @@ package search import ( - "github.com/ewhal/nyaa/db" - "github.com/ewhal/nyaa/model" - "github.com/ewhal/nyaa/service/torrent" - "github.com/ewhal/nyaa/util/log" "net/http" "strconv" "strings" "unicode" "unicode/utf8" + + "github.com/ewhal/nyaa/cache" + "github.com/ewhal/nyaa/common" + "github.com/ewhal/nyaa/db" + "github.com/ewhal/nyaa/model" + "github.com/ewhal/nyaa/service/torrent" + "github.com/ewhal/nyaa/util/log" ) -type Status uint8 - -const ( - ShowAll Status = iota - FilterRemakes - Trusted - APlus -) - -type SortMode uint8 - -const ( - ID SortMode = iota - Name - Date - Downloads - Size -) - -type Category struct { - Main, Sub uint8 -} - -func (c Category) String() (s string) { - if c.Main != 0 { - s += strconv.Itoa(int(c.Main)) - } - s += "_" - if c.Sub != 0 { - s += strconv.Itoa(int(c.Sub)) - } - return -} - -type SearchParam struct { - Order bool // True means acsending - Status Status - Sort SortMode - Category Category - Max uint - Query string -} - -func SearchByQuery(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, count int, err error) { +func SearchByQuery(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, count int, err error) { search, tor, count, err = searchByQuery(r, pagenum, true) return } -func SearchByQueryNoCount(r *http.Request, pagenum int) (search SearchParam, tor []model.Torrent, err error) { +func SearchByQueryNoCount(r *http.Request, pagenum int) (search common.SearchParam, tor []model.Torrent, err error) { search, tor, _, err = searchByQuery(r, pagenum, false) return } -func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchParam, tor []model.Torrent, count int, err error) { +func searchByQuery(r *http.Request, pagenum int, countAll bool) ( + search common.SearchParam, tor []model.Torrent, count int, err error, +) { max, err := strconv.ParseUint(r.URL.Query().Get("max"), 10, 32) if err != nil { max = 50 // default Value maxPerPage @@ -74,15 +36,16 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa } search.Max = uint(max) + search.Page = pagenum search.Query = r.URL.Query().Get("q") switch s := r.URL.Query().Get("s"); s { case "1": - search.Status = FilterRemakes + search.Status = common.FilterRemakes case "2": - search.Status = Trusted + search.Status = common.Trusted case "3": - search.Status = APlus + search.Status = common.APlus } catString := r.URL.Query().Get("c") @@ -107,16 +70,16 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa switch s := r.URL.Query().Get("sort"); s { case "1": - search.Sort = Name + search.Sort = common.Name orderBy += "torrent_name" case "2": - search.Sort = Date + search.Sort = common.Date orderBy += "date" case "3": - search.Sort = Downloads + search.Sort = common.Downloads orderBy += "downloads" case "4": - search.Sort = Size + search.Sort = common.Size orderBy += "filesize" default: orderBy += "torrent_id" @@ -132,66 +95,63 @@ func searchByQuery(r *http.Request, pagenum int, countAll bool) (search SearchPa orderBy += "desc" } - userID := r.URL.Query().Get("userID") + tor, count, err = cache.Get(search, func() (tor []model.Torrent, count int, err error) { + parameters := torrentService.WhereParams{ + Params: make([]interface{}, 0, 64), + } + conditions := make([]string, 0, 64) + if search.Category.Main != 0 { + conditions = append(conditions, "category = ?") + parameters.Params = append(parameters.Params, string(catString[0])) + } + if search.Category.Sub != 0 { + conditions = append(conditions, "sub_category = ?") + parameters.Params = append(parameters.Params, string(catString[2])) + } + if search.Status != 0 { + if search.Status == 3 { + conditions = append(conditions, "status != ?") + } else { + conditions = append(conditions, "status = ?") + } + parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1)) + } - parameters := torrentService.WhereParams{ - Params: make([]interface{}, 0, 64), - } - conditions := make([]string, 0, 64) - if search.Category.Main != 0 { - conditions = append(conditions, "category = ?") - parameters.Params = append(parameters.Params, string(catString[0])) - } - if search.Category.Sub != 0 { - conditions = append(conditions, "sub_category = ?") - parameters.Params = append(parameters.Params, string(catString[2])) - } - if userID != "" { - conditions = append(conditions, "uploader = ?") - parameters.Params = append(parameters.Params, userID) - } - if search.Status != 0 { - if search.Status == 3 { - conditions = append(conditions, "status != ?") + searchQuerySplit := strings.Fields(search.Query) + for i, word := range searchQuerySplit { + firstRune, _ := utf8.DecodeRuneInString(word) + if len(word) == 1 && unicode.IsPunct(firstRune) { + // some queries have a single punctuation character + // which causes a full scan instead of using the index + // and yields no meaningful results. + // due to len() == 1 we're just looking at 1-byte/ascii + // punctuation characters. + continue + } + + // SQLite has case-insensitive LIKE, but no ILIKE + var operator string + if db.ORM.Dialect().GetName() == "sqlite3" { + operator = "LIKE ?" + } else { + operator = "ILIKE ?" + } + + // TODO: make this faster ? + conditions = append(conditions, "torrent_name "+operator) + parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") + } + + parameters.Conditions = strings.Join(conditions[:], " AND ") + log.Infof("SQL query is :: %s\n", parameters.Conditions) + if countAll { + tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) } else { - conditions = append(conditions, "status = ?") - } - parameters.Params = append(parameters.Params, strconv.Itoa(int(search.Status)+1)) - } - - searchQuerySplit := strings.Fields(search.Query) - for i, word := range searchQuerySplit { - firstRune, _ := utf8.DecodeRuneInString(word) - if len(word) == 1 && unicode.IsPunct(firstRune) { - // some queries have a single punctuation character - // which causes a full scan instead of using the index - // and yields no meaningful results. - // due to len() == 1 we're just looking at 1-byte/ascii - // punctuation characters. - continue + tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(search.Page-1)) } - // TEMP: Workaround to at least make SQLite search testable for - // development. - // TODO: Actual case-insensitive search for SQLite - var operator string - if db.ORM.Dialect().GetName() == "sqlite3" { - operator = "LIKE ?" - } else { - operator = "ILIKE ?" - } + return + }) - // TODO: make this faster ? - conditions = append(conditions, "torrent_name "+operator) - parameters.Params = append(parameters.Params, "%"+searchQuerySplit[i]+"%") - } - - parameters.Conditions = strings.Join(conditions[:], " AND ") - log.Infof("SQL query is :: %s\n", parameters.Conditions) - if countAll { - tor, count, err = torrentService.GetTorrentsOrderBy(¶meters, orderBy, int(search.Max), int(search.Max)*(pagenum-1)) - } else { - tor, err = torrentService.GetTorrentsOrderByNoCount(¶meters, orderBy, int(search.Max), int(search.Max)*(pagenum-1)) - } return }