Framework for caching []model.Torrents
Cette révision appartient à :
Parent
839bc46163
révision
7ff8e97c9d
4 fichiers modifiés avec 205 ajouts et 38 suppressions
125
cache/cache.go
externe
Fichier normal
125
cache/cache.go
externe
Fichier normal
|
@ -0,0 +1,125 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ewhal/nyaa/model"
|
||||
"github.com/ewhal/nyaa/util/search"
|
||||
)
|
||||
|
||||
var (
|
||||
cache = make(map[search.SearchParam]*list.Element, 10)
|
||||
ll = list.New()
|
||||
totalUsed int
|
||||
mu sync.Mutex
|
||||
|
||||
// Mutable for quicker testing
|
||||
expiryTime = time.Second * 60
|
||||
|
||||
// Size sets the maximum size of the cache before evicting unread data in MB
|
||||
Size float64 = 1 << 10
|
||||
)
|
||||
|
||||
// Key stores the ID of either a thread or board page
|
||||
type Key struct {
|
||||
LastN uint8
|
||||
Board string
|
||||
ID uint64
|
||||
}
|
||||
|
||||
// Single cache entry
|
||||
type store struct {
|
||||
// Controls general access to the contents of the struct, except for size
|
||||
sync.Mutex
|
||||
lastFetched time.Time
|
||||
key search.SearchParam
|
||||
data []model.Torrents
|
||||
size int
|
||||
}
|
||||
|
||||
// Check the cache for and existing record. If miss, run fn to retrieve fresh
|
||||
// values.
|
||||
func CheckCache(key search.SearchParam, fn func() ([]model.Torrents, error)) (
|
||||
[]model.Torrents, error,
|
||||
) {
|
||||
s := getStore(key)
|
||||
|
||||
// Also keeps multiple requesters from simultaneously requesting the same
|
||||
// data
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
if s.isFresh() {
|
||||
return s.data, nil
|
||||
}
|
||||
|
||||
data, err := fn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.lastFetched = time.Now()
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Retrieve a store from the cache or create a new one
|
||||
func getStore(k search.SearchParam) (s *store) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
el := cache[k]
|
||||
if el == nil {
|
||||
s = &store{key: k}
|
||||
cache[k] = ll.PushFront(s)
|
||||
} else {
|
||||
ll.MoveToFront(el)
|
||||
s = el.Value.(*store)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Clear the cache. Only used for testing.
|
||||
func Clear() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
ll = list.New()
|
||||
cache = make(map[search.SearchParam]*list.Element, 10)
|
||||
}
|
||||
|
||||
// Update the total used memory counter and evict, if over limit
|
||||
func updateUsedSize(delta int) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
totalUsed += delta
|
||||
|
||||
for totalUsed > int(Size)*(1<<20) {
|
||||
s := ll.Remove(ll.Back()).(*store)
|
||||
delete(cache, s.key)
|
||||
totalUsed -= s.size
|
||||
}
|
||||
}
|
||||
|
||||
// Return, if the data can still be considered fresh, without querying the DB
|
||||
func (s *store) isFresh() bool {
|
||||
return s.lastFetched.Add(expiryTime).Before(time.Now())
|
||||
}
|
||||
|
||||
// Stores the new values of s. Calculates and stores the new size. Passes the
|
||||
// delta to the central cache to fire eviction checks.
|
||||
func (s *store) update(data []model.Torrents) {
|
||||
newSize := 0
|
||||
for _, d := range data {
|
||||
newSize += d.Size()
|
||||
}
|
||||
s.data = data
|
||||
delta := newSize - s.size
|
||||
s.size = newSize
|
||||
|
||||
// Technically it is possible to update the size even when the store is
|
||||
// already evicted, but that should never happen, unless you have a very
|
||||
// small cache, very large stored datasets and a lot of traffic.
|
||||
updateUsedSize(delta)
|
||||
}
|
|
@ -8,21 +8,29 @@ type Comment struct {
|
|||
Id uint `gorm:"column:comment_id;primary_key"`
|
||||
TorrentId uint `gorm:"column:torrent_id"`
|
||||
UserId uint `gorm:"column:user_id"`
|
||||
Content string `gorm:"column:content"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at"`
|
||||
|
||||
Torrent *Torrents `gorm:"ForeignKey:torrent_id"`
|
||||
User *User `gorm:"ForeignKey:user_id"`
|
||||
Content string `gorm:"column:content"`
|
||||
}
|
||||
|
||||
// Returns the total size of memory recursively allocated for this struct
|
||||
func (c Comment) Size() int {
|
||||
return (3 + 3*2 + 2 + 2 + len(c.Content)) * 8
|
||||
}
|
||||
|
||||
type OldComment struct {
|
||||
TorrentId uint `gorm:"column:torrent_id"`
|
||||
Date time.Time `gorm:"column:date"`
|
||||
Torrent *Torrents `gorm:"ForeignKey:torrent_id"`
|
||||
Username string `gorm:"column:username"`
|
||||
Content string `gorm:"column:content"`
|
||||
Date time.Time `gorm:"column:date"`
|
||||
}
|
||||
|
||||
Torrent *Torrents `gorm:"ForeignKey:torrent_id"`
|
||||
// Returns the total size of memory recursively allocated for this struct
|
||||
func (c OldComment) Size() int {
|
||||
return (4 + 2*2 + len(c.Username) + len(c.Content)) * 8
|
||||
}
|
||||
|
||||
func (c OldComment) TableName() string {
|
||||
|
|
|
@ -20,23 +20,42 @@ type Feed struct {
|
|||
}
|
||||
|
||||
type Torrents struct {
|
||||
Id uint `gorm:"column:torrent_id;primary_key"`
|
||||
Name string `gorm:"column:torrent_name"`
|
||||
Hash string `gorm:"column:torrent_hash"`
|
||||
Category int `gorm:"column:category"`
|
||||
Sub_Category int `gorm:"column:sub_category"`
|
||||
Status int `gorm:"column:status"`
|
||||
Date time.Time `gorm:"column:date"`
|
||||
UploaderId uint `gorm:"column:uploader"`
|
||||
Downloads int `gorm:"column:downloads"`
|
||||
Stardom int `gorm:"column:stardom"`
|
||||
Filesize int64 `gorm:"column:filesize"`
|
||||
Description string `gorm:"column:description"`
|
||||
WebsiteLink string `gorm:"column:website_link"`
|
||||
Category int `gorm:"column:category"`
|
||||
Status int `gorm:"column:status"`
|
||||
Sub_Category int `gorm:"column:sub_category"`
|
||||
UploaderId uint `gorm:"column:uploader"`
|
||||
Downloads int `gorm:"column:downloads"`
|
||||
Stardom int `gorm:"column:stardom"`
|
||||
Filesize int64 `gorm:"column:filesize"`
|
||||
Id uint `gorm:"column:torrent_id;primary_key"`
|
||||
Date time.Time `gorm:"column:date"`
|
||||
Uploader *User `gorm:"ForeignKey:uploader"`
|
||||
Name string `gorm:"column:torrent_name"`
|
||||
Hash string `gorm:"column:torrent_hash"`
|
||||
Description string `gorm:"column:description"`
|
||||
WebsiteLink string `gorm:"column:website_link"`
|
||||
OldComments []OldComment `gorm:"ForeignKey:torrent_id"`
|
||||
Comments []Comment `gorm:"ForeignKey:torrent_id"`
|
||||
}
|
||||
|
||||
Uploader *User `gorm:"ForeignKey:uploader"`
|
||||
OldComments []OldComment `gorm:"ForeignKey:torrent_id"`
|
||||
Comments []Comment `gorm:"ForeignKey:torrent_id"`
|
||||
// Returns the total size of memory recursively allocated for this struct
|
||||
func (t Torrents) Size() (s int) {
|
||||
s += 12 + // numbers and pointers
|
||||
4*2 + // string pointer sizes
|
||||
// string array sizes
|
||||
len(t.Name) + len(t.Hash) + len(t.Description) + len(t.WebsiteLink) +
|
||||
2*2 // array pointer length
|
||||
s *= 8 // Assume 64 bit OS
|
||||
|
||||
s += t.Uploader.Size()
|
||||
for _, c := range t.OldComments {
|
||||
s += c.Size()
|
||||
}
|
||||
for _, c := range t.Comments {
|
||||
s += c.Size()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
/* We need JSON Object instead because of Magnet URL that is not in the database but generated dynamically */
|
||||
|
@ -54,20 +73,20 @@ type CommentsJson struct {
|
|||
}
|
||||
|
||||
type TorrentsJson struct {
|
||||
Status int `json:"status"`
|
||||
Downloads int `json:"downloads"`
|
||||
UploaderId uint `json:"uploader_id"`
|
||||
Id string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Status int `json:"status"`
|
||||
Hash string `json:"hash"`
|
||||
Date string `json:"date"`
|
||||
Filesize string `json:"filesize"`
|
||||
Description template.HTML `json:"description"`
|
||||
Comments []CommentsJson `json:"comments"`
|
||||
Sub_Category string `json:"sub_category"`
|
||||
Category string `json:"category"`
|
||||
Downloads int `json:"downloads"`
|
||||
UploaderId uint `json:"uploader_id"`
|
||||
Description template.HTML `json:"description"`
|
||||
WebsiteLink template.URL `json:"website_link"`
|
||||
Magnet template.URL `json:"magnet"`
|
||||
Comments []CommentsJson `json:"comments"`
|
||||
}
|
||||
|
||||
/* Model Conversion to Json */
|
||||
|
|
|
@ -8,14 +8,14 @@ import (
|
|||
type omit bool
|
||||
|
||||
type User struct {
|
||||
Id uint `gorm:"column:user_id;primary_key"`
|
||||
Username string `gorm:"column:username"`
|
||||
Password string `gorm:"column:password"`
|
||||
Email string `gorm:"column:email"`
|
||||
Status int `gorm:"column:status"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at"`
|
||||
/*Api*/Token string `gorm:"column:api_token"`
|
||||
Id uint `gorm:"column:user_id;primary_key"`
|
||||
Username string `gorm:"column:username"`
|
||||
Password string `gorm:"column:password"`
|
||||
Email string `gorm:"column:email"`
|
||||
Status int `gorm:"column:status"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at"`
|
||||
/*Api*/ Token string `gorm:"column:api_token"`
|
||||
//ApiTokenExpiry
|
||||
|
||||
// Liking
|
||||
|
@ -24,14 +24,29 @@ type User struct {
|
|||
Likings []User `gorm:"foreignkey:userId;associationforeignkey:follower_id;many2many:users_followers;"`
|
||||
Liked []User `gorm:"foreignkey:follower_id;associationforeignkey:userId;many2many:users_followers;"`
|
||||
|
||||
Md5 string `json:"md5"`
|
||||
TokenExpiration time.Time `gorm:"column:api_token_expiry"`
|
||||
Language string `gorm:"column:language"`
|
||||
Torrents []Torrents `gorm:"ForeignKey:UploaderId"`
|
||||
Md5 string `json:"md5"`
|
||||
TokenExpiration time.Time `gorm:"column:api_token_expiry"`
|
||||
Language string `gorm:"column:language"`
|
||||
Torrents []Torrents `gorm:"ForeignKey:UploaderId"`
|
||||
}
|
||||
|
||||
// Returns the total size of memory recursively allocated for this struct
|
||||
func (u User) Size() (s int) {
|
||||
s += 4 + // ints
|
||||
6*2 + // string pointers
|
||||
4*3 + //time.Time
|
||||
3*2 + // arrays
|
||||
// string arrays
|
||||
len(u.Username) + len(u.Password) + len(u.Email) + len(u.Token) + len(u.Md5) + len(u.Language)
|
||||
s *= 8
|
||||
|
||||
// Ignoring foreign key users. Fuck them.
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type PublicUser struct {
|
||||
User *User
|
||||
User *User
|
||||
}
|
||||
|
||||
// UsersFollowers is a relation table to relate users each other.
|
||||
|
|
Référencer dans un nouveau ticket