Framework for caching []model.Torrents

cache/cache.go

@@ -0,0 +1,125 @@
+package cache
+
+import (
+	"container/list"
+	"sync"
+	"time"
+
+	"github.com/ewhal/nyaa/model"
+	"github.com/ewhal/nyaa/util/search"
+)
+
+var (
+	cache     = make(map[search.SearchParam]*list.Element, 10)
+	ll        = list.New()
+	totalUsed int
+	mu        sync.Mutex
+
+	// Mutable for quicker testing
+	expiryTime = time.Second * 60
+
+	// Size sets the maximum size of the cache before evicting unread data in MB
+	Size float64 = 1 << 10
+)
+
+// Key stores the ID of either a thread or board page
+type Key struct {
+	LastN uint8
+	Board string
+	ID    uint64
+}
+
+// Single cache entry
+type store struct {
+	// Controls general access to the contents of the struct, except for size
+	sync.Mutex
+	lastFetched time.Time
+	key         search.SearchParam
+	data        []model.Torrents
+	size        int
+}
+
+// Check the cache for and existing record. If miss, run fn to retrieve fresh
+// values.
+func CheckCache(key search.SearchParam, fn func() ([]model.Torrents, error)) (
+	[]model.Torrents, error,
+) {
+	s := getStore(key)
+
+	// Also keeps multiple requesters from simultaneously requesting the same
+	// data
+	s.Lock()
+	defer s.Unlock()
+
+	if s.isFresh() {
+		return s.data, nil
+	}
+
+	data, err := fn()
+	if err != nil {
+		return nil, err
+	}
+	s.lastFetched = time.Now()
+	return data, nil
+}
+
+// Retrieve a store from the cache or create a new one
+func getStore(k search.SearchParam) (s *store) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	el := cache[k]
+	if el == nil {
+		s = &store{key: k}
+		cache[k] = ll.PushFront(s)
+	} else {
+		ll.MoveToFront(el)
+		s = el.Value.(*store)
+	}
+	return s
+}
+
+// Clear the cache. Only used for testing.
+func Clear() {
+	mu.Lock()
+	defer mu.Unlock()
+
+	ll = list.New()
+	cache = make(map[search.SearchParam]*list.Element, 10)
+}
+
+// Update the total used memory counter and evict, if over limit
+func updateUsedSize(delta int) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	totalUsed += delta
+
+	for totalUsed > int(Size)*(1<<20) {
+		s := ll.Remove(ll.Back()).(*store)
+		delete(cache, s.key)
+		totalUsed -= s.size
+	}
+}
+
+// Return, if the data can still be considered fresh, without querying the DB
+func (s *store) isFresh() bool {
+	return s.lastFetched.Add(expiryTime).Before(time.Now())
+}
+
+// Stores the new values of s. Calculates and stores the new size. Passes the
+// delta to the central cache to fire eviction checks.
+func (s *store) update(data []model.Torrents) {
+	newSize := 0
+	for _, d := range data {
+		newSize += d.Size()
+	}
+	s.data = data
+	delta := newSize - s.size
+	s.size = newSize
+
+	// Technically it is possible to update the size even when the store is
+	// already evicted, but that should never happen, unless you have a very
+	// small cache, very large stored datasets and a lot of traffic.
+	updateUsedSize(delta)
+}

model/comment.go

@@ -8,21 +8,29 @@ type Comment struct {
 	Id        uint      `gorm:"column:comment_id;primary_key"`
 	TorrentId uint      `gorm:"column:torrent_id"`
 	UserId    uint      `gorm:"column:user_id"`
-	Content   string    `gorm:"column:content"`
 	CreatedAt time.Time `gorm:"column:created_at"`
 	UpdatedAt time.Time `gorm:"column:updated_at"`
-
 	Torrent   *Torrents `gorm:"ForeignKey:torrent_id"`
 	User      *User     `gorm:"ForeignKey:user_id"`
+	Content   string    `gorm:"column:content"`
+}
+
+// Returns the total size of memory recursively allocated for this struct
+func (c Comment) Size() int {
+	return (3 + 3*2 + 2 + 2 + len(c.Content)) * 8
 }
 
 type OldComment struct {
 	TorrentId uint      `gorm:"column:torrent_id"`
+	Date      time.Time `gorm:"column:date"`
+	Torrent   *Torrents `gorm:"ForeignKey:torrent_id"`
 	Username  string    `gorm:"column:username"`
 	Content   string    `gorm:"column:content"`
-	Date      time.Time `gorm:"column:date"`
+}
 
-	Torrent   *Torrents `gorm:"ForeignKey:torrent_id"`
+// Returns the total size of memory recursively allocated for this struct
+func (c OldComment) Size() int {
+	return (4 + 2*2 + len(c.Username) + len(c.Content)) * 8
 }
 
 func (c OldComment) TableName() string {

model/torrent.go

@@ -20,23 +20,42 @@ type Feed struct {
 }
 
 type Torrents struct {
-	Id           uint      `gorm:"column:torrent_id;primary_key"`
+	Category     int          `gorm:"column:category"`
-	Name         string    `gorm:"column:torrent_name"`
+	Status       int          `gorm:"column:status"`
-	Hash         string    `gorm:"column:torrent_hash"`
+	Sub_Category int          `gorm:"column:sub_category"`
-	Category     int       `gorm:"column:category"`
+	UploaderId   uint         `gorm:"column:uploader"`
-	Sub_Category int       `gorm:"column:sub_category"`
+	Downloads    int          `gorm:"column:downloads"`
-	Status       int       `gorm:"column:status"`
+	Stardom      int          `gorm:"column:stardom"`
-	Date         time.Time `gorm:"column:date"`
+	Filesize     int64        `gorm:"column:filesize"`
-	UploaderId   uint      `gorm:"column:uploader"`
+	Id           uint         `gorm:"column:torrent_id;primary_key"`
-	Downloads    int       `gorm:"column:downloads"`
+	Date         time.Time    `gorm:"column:date"`
-	Stardom      int       `gorm:"column:stardom"`
+	Uploader     *User        `gorm:"ForeignKey:uploader"`
-	Filesize     int64     `gorm:"column:filesize"`
+	Name         string       `gorm:"column:torrent_name"`
-	Description  string    `gorm:"column:description"`
+	Hash         string       `gorm:"column:torrent_hash"`
-	WebsiteLink  string    `gorm:"column:website_link"`
+	Description  string       `gorm:"column:description"`
+	WebsiteLink  string       `gorm:"column:website_link"`
+	OldComments  []OldComment `gorm:"ForeignKey:torrent_id"`
+	Comments     []Comment    `gorm:"ForeignKey:torrent_id"`
+}
 
-	Uploader    *User        `gorm:"ForeignKey:uploader"`
+// Returns the total size of memory recursively allocated for this struct
-	OldComments []OldComment `gorm:"ForeignKey:torrent_id"`
+func (t Torrents) Size() (s int) {
-	Comments    []Comment    `gorm:"ForeignKey:torrent_id"`
+	s += 12 + // numbers and pointers
+		4*2 + // string pointer sizes
+		// string array sizes
+		len(t.Name) + len(t.Hash) + len(t.Description) + len(t.WebsiteLink) +
+		2*2 // array pointer length
+	s *= 8 // Assume 64 bit OS
+
+	s += t.Uploader.Size()
+	for _, c := range t.OldComments {
+		s += c.Size()
+	}
+	for _, c := range t.Comments {
+		s += c.Size()
+	}
+
+	return
 }
 
 /* We need JSON Object instead because of Magnet URL that is not in the database but generated dynamically */
@@ -54,20 +73,20 @@ type CommentsJson struct {
 }
 
 type TorrentsJson struct {
+	Status       int            `json:"status"`
+	Downloads    int            `json:"downloads"`
+	UploaderId   uint           `json:"uploader_id"`
 	Id           string         `json:"id"`
 	Name         string         `json:"name"`
-	Status       int            `json:"status"`
 	Hash         string         `json:"hash"`
 	Date         string         `json:"date"`
 	Filesize     string         `json:"filesize"`
-	Description  template.HTML  `json:"description"`
-	Comments     []CommentsJson `json:"comments"`
 	Sub_Category string         `json:"sub_category"`
 	Category     string         `json:"category"`
-	Downloads    int            `json:"downloads"`
+	Description  template.HTML  `json:"description"`
-	UploaderId   uint           `json:"uploader_id"`
 	WebsiteLink  template.URL   `json:"website_link"`
 	Magnet       template.URL   `json:"magnet"`
+	Comments     []CommentsJson `json:"comments"`
 }
 
 /* Model Conversion to Json */

model/user.go

@@ -8,14 +8,14 @@ import (
 type omit bool
 
 type User struct {
-	Id              uint      `gorm:"column:user_id;primary_key"`
+	Id        uint      `gorm:"column:user_id;primary_key"`
-	Username        string    `gorm:"column:username"`
+	Username  string    `gorm:"column:username"`
-	Password        string    `gorm:"column:password"`
+	Password  string    `gorm:"column:password"`
-	Email           string    `gorm:"column:email"`
+	Email     string    `gorm:"column:email"`
-	Status          int       `gorm:"column:status"`
+	Status    int       `gorm:"column:status"`
-	CreatedAt       time.Time `gorm:"column:created_at"`
+	CreatedAt time.Time `gorm:"column:created_at"`
-	UpdatedAt       time.Time `gorm:"column:updated_at"`
+	UpdatedAt time.Time `gorm:"column:updated_at"`
-	/*Api*/Token    string    `gorm:"column:api_token"`
+	/*Api*/ Token string `gorm:"column:api_token"`
 	//ApiTokenExpiry
 
 	// Liking
@@ -24,14 +24,29 @@ type User struct {
 	Likings     []User `gorm:"foreignkey:userId;associationforeignkey:follower_id;many2many:users_followers;"`
 	Liked       []User `gorm:"foreignkey:follower_id;associationforeignkey:userId;many2many:users_followers;"`
 
-	Md5 string `json:"md5"`
+	Md5             string     `json:"md5"`
-	TokenExpiration time.Time `gorm:"column:api_token_expiry"`
+	TokenExpiration time.Time  `gorm:"column:api_token_expiry"`
-	Language        string    `gorm:"column:language"`
+	Language        string     `gorm:"column:language"`
-	Torrents    []Torrents `gorm:"ForeignKey:UploaderId"`
+	Torrents        []Torrents `gorm:"ForeignKey:UploaderId"`
+}
+
+// Returns the total size of memory recursively allocated for this struct
+func (u User) Size() (s int) {
+	s += 4 + // ints
+		6*2 + // string pointers
+		4*3 + //time.Time
+		3*2 + // arrays
+		// string arrays
+		len(u.Username) + len(u.Password) + len(u.Email) + len(u.Token) + len(u.Md5) + len(u.Language)
+	s *= 8
+
+	// Ignoring foreign key users. Fuck them.
+
+	return
 }
 
 type PublicUser struct {
-	User      *User
+	User *User
 }
 
 // UsersFollowers is a relation table to relate users each other.