Albirew/nyaa-pantsu
Archivé
1
0
Bifurcation 0

Framework for caching []model.Torrents

Cette révision appartient à :
bakape 2017-05-10 11:27:17 +03:00
Parent 839bc46163
révision 7ff8e97c9d
4 fichiers modifiés avec 205 ajouts et 38 suppressions

125
cache/cache.go externe Fichier normal
Voir le fichier

@ -0,0 +1,125 @@
package cache
import (
"container/list"
"sync"
"time"
"github.com/ewhal/nyaa/model"
"github.com/ewhal/nyaa/util/search"
)
var (
cache = make(map[search.SearchParam]*list.Element, 10)
ll = list.New()
totalUsed int
mu sync.Mutex
// Mutable for quicker testing
expiryTime = time.Second * 60
// Size sets the maximum size of the cache before evicting unread data in MB
Size float64 = 1 << 10
)
// Key stores the ID of either a thread or board page
type Key struct {
LastN uint8
Board string
ID uint64
}
// Single cache entry
type store struct {
// Controls general access to the contents of the struct, except for size
sync.Mutex
lastFetched time.Time
key search.SearchParam
data []model.Torrents
size int
}
// Check the cache for and existing record. If miss, run fn to retrieve fresh
// values.
func CheckCache(key search.SearchParam, fn func() ([]model.Torrents, error)) (
[]model.Torrents, error,
) {
s := getStore(key)
// Also keeps multiple requesters from simultaneously requesting the same
// data
s.Lock()
defer s.Unlock()
if s.isFresh() {
return s.data, nil
}
data, err := fn()
if err != nil {
return nil, err
}
s.lastFetched = time.Now()
return data, nil
}
// Retrieve a store from the cache or create a new one
func getStore(k search.SearchParam) (s *store) {
mu.Lock()
defer mu.Unlock()
el := cache[k]
if el == nil {
s = &store{key: k}
cache[k] = ll.PushFront(s)
} else {
ll.MoveToFront(el)
s = el.Value.(*store)
}
return s
}
// Clear the cache. Only used for testing.
func Clear() {
mu.Lock()
defer mu.Unlock()
ll = list.New()
cache = make(map[search.SearchParam]*list.Element, 10)
}
// Update the total used memory counter and evict, if over limit
func updateUsedSize(delta int) {
mu.Lock()
defer mu.Unlock()
totalUsed += delta
for totalUsed > int(Size)*(1<<20) {
s := ll.Remove(ll.Back()).(*store)
delete(cache, s.key)
totalUsed -= s.size
}
}
// Return, if the data can still be considered fresh, without querying the DB
func (s *store) isFresh() bool {
return s.lastFetched.Add(expiryTime).Before(time.Now())
}
// Stores the new values of s. Calculates and stores the new size. Passes the
// delta to the central cache to fire eviction checks.
func (s *store) update(data []model.Torrents) {
newSize := 0
for _, d := range data {
newSize += d.Size()
}
s.data = data
delta := newSize - s.size
s.size = newSize
// Technically it is possible to update the size even when the store is
// already evicted, but that should never happen, unless you have a very
// small cache, very large stored datasets and a lot of traffic.
updateUsedSize(delta)
}

Voir le fichier

@ -8,21 +8,29 @@ type Comment struct {
Id uint `gorm:"column:comment_id;primary_key"`
TorrentId uint `gorm:"column:torrent_id"`
UserId uint `gorm:"column:user_id"`
Content string `gorm:"column:content"`
CreatedAt time.Time `gorm:"column:created_at"`
UpdatedAt time.Time `gorm:"column:updated_at"`
Torrent *Torrents `gorm:"ForeignKey:torrent_id"`
User *User `gorm:"ForeignKey:user_id"`
Content string `gorm:"column:content"`
}
// Returns the total size of memory recursively allocated for this struct
func (c Comment) Size() int {
return (3 + 3*2 + 2 + 2 + len(c.Content)) * 8
}
type OldComment struct {
TorrentId uint `gorm:"column:torrent_id"`
Date time.Time `gorm:"column:date"`
Torrent *Torrents `gorm:"ForeignKey:torrent_id"`
Username string `gorm:"column:username"`
Content string `gorm:"column:content"`
Date time.Time `gorm:"column:date"`
}
Torrent *Torrents `gorm:"ForeignKey:torrent_id"`
// Returns the total size of memory recursively allocated for this struct
func (c OldComment) Size() int {
return (4 + 2*2 + len(c.Username) + len(c.Content)) * 8
}
func (c OldComment) TableName() string {

Voir le fichier

@ -20,23 +20,42 @@ type Feed struct {
}
type Torrents struct {
Id uint `gorm:"column:torrent_id;primary_key"`
Name string `gorm:"column:torrent_name"`
Hash string `gorm:"column:torrent_hash"`
Category int `gorm:"column:category"`
Sub_Category int `gorm:"column:sub_category"`
Status int `gorm:"column:status"`
Date time.Time `gorm:"column:date"`
UploaderId uint `gorm:"column:uploader"`
Downloads int `gorm:"column:downloads"`
Stardom int `gorm:"column:stardom"`
Filesize int64 `gorm:"column:filesize"`
Description string `gorm:"column:description"`
WebsiteLink string `gorm:"column:website_link"`
Category int `gorm:"column:category"`
Status int `gorm:"column:status"`
Sub_Category int `gorm:"column:sub_category"`
UploaderId uint `gorm:"column:uploader"`
Downloads int `gorm:"column:downloads"`
Stardom int `gorm:"column:stardom"`
Filesize int64 `gorm:"column:filesize"`
Id uint `gorm:"column:torrent_id;primary_key"`
Date time.Time `gorm:"column:date"`
Uploader *User `gorm:"ForeignKey:uploader"`
Name string `gorm:"column:torrent_name"`
Hash string `gorm:"column:torrent_hash"`
Description string `gorm:"column:description"`
WebsiteLink string `gorm:"column:website_link"`
OldComments []OldComment `gorm:"ForeignKey:torrent_id"`
Comments []Comment `gorm:"ForeignKey:torrent_id"`
}
Uploader *User `gorm:"ForeignKey:uploader"`
OldComments []OldComment `gorm:"ForeignKey:torrent_id"`
Comments []Comment `gorm:"ForeignKey:torrent_id"`
// Returns the total size of memory recursively allocated for this struct
func (t Torrents) Size() (s int) {
s += 12 + // numbers and pointers
4*2 + // string pointer sizes
// string array sizes
len(t.Name) + len(t.Hash) + len(t.Description) + len(t.WebsiteLink) +
2*2 // array pointer length
s *= 8 // Assume 64 bit OS
s += t.Uploader.Size()
for _, c := range t.OldComments {
s += c.Size()
}
for _, c := range t.Comments {
s += c.Size()
}
return
}
/* We need JSON Object instead because of Magnet URL that is not in the database but generated dynamically */
@ -54,20 +73,20 @@ type CommentsJson struct {
}
type TorrentsJson struct {
Status int `json:"status"`
Downloads int `json:"downloads"`
UploaderId uint `json:"uploader_id"`
Id string `json:"id"`
Name string `json:"name"`
Status int `json:"status"`
Hash string `json:"hash"`
Date string `json:"date"`
Filesize string `json:"filesize"`
Description template.HTML `json:"description"`
Comments []CommentsJson `json:"comments"`
Sub_Category string `json:"sub_category"`
Category string `json:"category"`
Downloads int `json:"downloads"`
UploaderId uint `json:"uploader_id"`
Description template.HTML `json:"description"`
WebsiteLink template.URL `json:"website_link"`
Magnet template.URL `json:"magnet"`
Comments []CommentsJson `json:"comments"`
}
/* Model Conversion to Json */

Voir le fichier

@ -8,14 +8,14 @@ import (
type omit bool
type User struct {
Id uint `gorm:"column:user_id;primary_key"`
Username string `gorm:"column:username"`
Password string `gorm:"column:password"`
Email string `gorm:"column:email"`
Status int `gorm:"column:status"`
CreatedAt time.Time `gorm:"column:created_at"`
UpdatedAt time.Time `gorm:"column:updated_at"`
/*Api*/Token string `gorm:"column:api_token"`
Id uint `gorm:"column:user_id;primary_key"`
Username string `gorm:"column:username"`
Password string `gorm:"column:password"`
Email string `gorm:"column:email"`
Status int `gorm:"column:status"`
CreatedAt time.Time `gorm:"column:created_at"`
UpdatedAt time.Time `gorm:"column:updated_at"`
/*Api*/ Token string `gorm:"column:api_token"`
//ApiTokenExpiry
// Liking
@ -24,14 +24,29 @@ type User struct {
Likings []User `gorm:"foreignkey:userId;associationforeignkey:follower_id;many2many:users_followers;"`
Liked []User `gorm:"foreignkey:follower_id;associationforeignkey:userId;many2many:users_followers;"`
Md5 string `json:"md5"`
TokenExpiration time.Time `gorm:"column:api_token_expiry"`
Language string `gorm:"column:language"`
Torrents []Torrents `gorm:"ForeignKey:UploaderId"`
Md5 string `json:"md5"`
TokenExpiration time.Time `gorm:"column:api_token_expiry"`
Language string `gorm:"column:language"`
Torrents []Torrents `gorm:"ForeignKey:UploaderId"`
}
// Returns the total size of memory recursively allocated for this struct
func (u User) Size() (s int) {
s += 4 + // ints
6*2 + // string pointers
4*3 + //time.Time
3*2 + // arrays
// string arrays
len(u.Username) + len(u.Password) + len(u.Email) + len(u.Token) + len(u.Md5) + len(u.Language)
s *= 8
// Ignoring foreign key users. Fuck them.
return
}
type PublicUser struct {
User *User
User *User
}
// UsersFollowers is a relation table to relate users each other.