diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 00000000..640f0fef --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,125 @@ +package cache + +import ( + "container/list" + "sync" + "time" + + "github.com/ewhal/nyaa/model" + "github.com/ewhal/nyaa/util/search" +) + +var ( + cache = make(map[search.SearchParam]*list.Element, 10) + ll = list.New() + totalUsed int + mu sync.Mutex + + // Mutable for quicker testing + expiryTime = time.Second * 60 + + // Size sets the maximum size of the cache before evicting unread data in MB + Size float64 = 1 << 10 +) + +// Key stores the ID of either a thread or board page +type Key struct { + LastN uint8 + Board string + ID uint64 +} + +// Single cache entry +type store struct { + // Controls general access to the contents of the struct, except for size + sync.Mutex + lastFetched time.Time + key search.SearchParam + data []model.Torrents + size int +} + +// Check the cache for and existing record. If miss, run fn to retrieve fresh +// values. +func CheckCache(key search.SearchParam, fn func() ([]model.Torrents, error)) ( + []model.Torrents, error, +) { + s := getStore(key) + + // Also keeps multiple requesters from simultaneously requesting the same + // data + s.Lock() + defer s.Unlock() + + if s.isFresh() { + return s.data, nil + } + + data, err := fn() + if err != nil { + return nil, err + } + s.lastFetched = time.Now() + return data, nil +} + +// Retrieve a store from the cache or create a new one +func getStore(k search.SearchParam) (s *store) { + mu.Lock() + defer mu.Unlock() + + el := cache[k] + if el == nil { + s = &store{key: k} + cache[k] = ll.PushFront(s) + } else { + ll.MoveToFront(el) + s = el.Value.(*store) + } + return s +} + +// Clear the cache. Only used for testing. +func Clear() { + mu.Lock() + defer mu.Unlock() + + ll = list.New() + cache = make(map[search.SearchParam]*list.Element, 10) +} + +// Update the total used memory counter and evict, if over limit +func updateUsedSize(delta int) { + mu.Lock() + defer mu.Unlock() + + totalUsed += delta + + for totalUsed > int(Size)*(1<<20) { + s := ll.Remove(ll.Back()).(*store) + delete(cache, s.key) + totalUsed -= s.size + } +} + +// Return, if the data can still be considered fresh, without querying the DB +func (s *store) isFresh() bool { + return s.lastFetched.Add(expiryTime).Before(time.Now()) +} + +// Stores the new values of s. Calculates and stores the new size. Passes the +// delta to the central cache to fire eviction checks. +func (s *store) update(data []model.Torrents) { + newSize := 0 + for _, d := range data { + newSize += d.Size() + } + s.data = data + delta := newSize - s.size + s.size = newSize + + // Technically it is possible to update the size even when the store is + // already evicted, but that should never happen, unless you have a very + // small cache, very large stored datasets and a lot of traffic. + updateUsedSize(delta) +} diff --git a/model/comment.go b/model/comment.go index 1d64a90d..b0bbfe2b 100644 --- a/model/comment.go +++ b/model/comment.go @@ -8,21 +8,29 @@ type Comment struct { Id uint `gorm:"column:comment_id;primary_key"` TorrentId uint `gorm:"column:torrent_id"` UserId uint `gorm:"column:user_id"` - Content string `gorm:"column:content"` CreatedAt time.Time `gorm:"column:created_at"` UpdatedAt time.Time `gorm:"column:updated_at"` - Torrent *Torrents `gorm:"ForeignKey:torrent_id"` User *User `gorm:"ForeignKey:user_id"` + Content string `gorm:"column:content"` +} + +// Returns the total size of memory recursively allocated for this struct +func (c Comment) Size() int { + return (3 + 3*2 + 2 + 2 + len(c.Content)) * 8 } type OldComment struct { TorrentId uint `gorm:"column:torrent_id"` + Date time.Time `gorm:"column:date"` + Torrent *Torrents `gorm:"ForeignKey:torrent_id"` Username string `gorm:"column:username"` Content string `gorm:"column:content"` - Date time.Time `gorm:"column:date"` +} - Torrent *Torrents `gorm:"ForeignKey:torrent_id"` +// Returns the total size of memory recursively allocated for this struct +func (c OldComment) Size() int { + return (4 + 2*2 + len(c.Username) + len(c.Content)) * 8 } func (c OldComment) TableName() string { diff --git a/model/torrent.go b/model/torrent.go index 1ed3dd95..ae5a7b4b 100644 --- a/model/torrent.go +++ b/model/torrent.go @@ -20,23 +20,42 @@ type Feed struct { } type Torrents struct { - Id uint `gorm:"column:torrent_id;primary_key"` - Name string `gorm:"column:torrent_name"` - Hash string `gorm:"column:torrent_hash"` - Category int `gorm:"column:category"` - Sub_Category int `gorm:"column:sub_category"` - Status int `gorm:"column:status"` - Date time.Time `gorm:"column:date"` - UploaderId uint `gorm:"column:uploader"` - Downloads int `gorm:"column:downloads"` - Stardom int `gorm:"column:stardom"` - Filesize int64 `gorm:"column:filesize"` - Description string `gorm:"column:description"` - WebsiteLink string `gorm:"column:website_link"` + Category int `gorm:"column:category"` + Status int `gorm:"column:status"` + Sub_Category int `gorm:"column:sub_category"` + UploaderId uint `gorm:"column:uploader"` + Downloads int `gorm:"column:downloads"` + Stardom int `gorm:"column:stardom"` + Filesize int64 `gorm:"column:filesize"` + Id uint `gorm:"column:torrent_id;primary_key"` + Date time.Time `gorm:"column:date"` + Uploader *User `gorm:"ForeignKey:uploader"` + Name string `gorm:"column:torrent_name"` + Hash string `gorm:"column:torrent_hash"` + Description string `gorm:"column:description"` + WebsiteLink string `gorm:"column:website_link"` + OldComments []OldComment `gorm:"ForeignKey:torrent_id"` + Comments []Comment `gorm:"ForeignKey:torrent_id"` +} - Uploader *User `gorm:"ForeignKey:uploader"` - OldComments []OldComment `gorm:"ForeignKey:torrent_id"` - Comments []Comment `gorm:"ForeignKey:torrent_id"` +// Returns the total size of memory recursively allocated for this struct +func (t Torrents) Size() (s int) { + s += 12 + // numbers and pointers + 4*2 + // string pointer sizes + // string array sizes + len(t.Name) + len(t.Hash) + len(t.Description) + len(t.WebsiteLink) + + 2*2 // array pointer length + s *= 8 // Assume 64 bit OS + + s += t.Uploader.Size() + for _, c := range t.OldComments { + s += c.Size() + } + for _, c := range t.Comments { + s += c.Size() + } + + return } /* We need JSON Object instead because of Magnet URL that is not in the database but generated dynamically */ @@ -54,20 +73,20 @@ type CommentsJson struct { } type TorrentsJson struct { + Status int `json:"status"` + Downloads int `json:"downloads"` + UploaderId uint `json:"uploader_id"` Id string `json:"id"` Name string `json:"name"` - Status int `json:"status"` Hash string `json:"hash"` Date string `json:"date"` Filesize string `json:"filesize"` - Description template.HTML `json:"description"` - Comments []CommentsJson `json:"comments"` Sub_Category string `json:"sub_category"` Category string `json:"category"` - Downloads int `json:"downloads"` - UploaderId uint `json:"uploader_id"` + Description template.HTML `json:"description"` WebsiteLink template.URL `json:"website_link"` Magnet template.URL `json:"magnet"` + Comments []CommentsJson `json:"comments"` } /* Model Conversion to Json */ diff --git a/model/user.go b/model/user.go index 829243d2..df63829a 100644 --- a/model/user.go +++ b/model/user.go @@ -8,14 +8,14 @@ import ( type omit bool type User struct { - Id uint `gorm:"column:user_id;primary_key"` - Username string `gorm:"column:username"` - Password string `gorm:"column:password"` - Email string `gorm:"column:email"` - Status int `gorm:"column:status"` - CreatedAt time.Time `gorm:"column:created_at"` - UpdatedAt time.Time `gorm:"column:updated_at"` - /*Api*/Token string `gorm:"column:api_token"` + Id uint `gorm:"column:user_id;primary_key"` + Username string `gorm:"column:username"` + Password string `gorm:"column:password"` + Email string `gorm:"column:email"` + Status int `gorm:"column:status"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` + /*Api*/ Token string `gorm:"column:api_token"` //ApiTokenExpiry // Liking @@ -24,14 +24,29 @@ type User struct { Likings []User `gorm:"foreignkey:userId;associationforeignkey:follower_id;many2many:users_followers;"` Liked []User `gorm:"foreignkey:follower_id;associationforeignkey:userId;many2many:users_followers;"` - Md5 string `json:"md5"` - TokenExpiration time.Time `gorm:"column:api_token_expiry"` - Language string `gorm:"column:language"` - Torrents []Torrents `gorm:"ForeignKey:UploaderId"` + Md5 string `json:"md5"` + TokenExpiration time.Time `gorm:"column:api_token_expiry"` + Language string `gorm:"column:language"` + Torrents []Torrents `gorm:"ForeignKey:UploaderId"` +} + +// Returns the total size of memory recursively allocated for this struct +func (u User) Size() (s int) { + s += 4 + // ints + 6*2 + // string pointers + 4*3 + //time.Time + 3*2 + // arrays + // string arrays + len(u.Username) + len(u.Password) + len(u.Email) + len(u.Token) + len(u.Md5) + len(u.Language) + s *= 8 + + // Ignoring foreign key users. Fuck them. + + return } type PublicUser struct { - User *User + User *User } // UsersFollowers is a relation table to relate users each other.