nyaa-pantsu/models/torrent.go

package models

import (
	"context"
	"errors"
	"fmt"
	"html/template"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"time"

	elastic "gopkg.in/olivere/elastic.v5"

	"net/http"
	"net/url"

	"github.com/NyaaPantsu/nyaa/config"
	"github.com/NyaaPantsu/nyaa/utils/cache"
	"github.com/NyaaPantsu/nyaa/utils/format"
	"github.com/NyaaPantsu/nyaa/utils/log"
	"github.com/NyaaPantsu/nyaa/utils/sanitize"
	"github.com/bradfitz/slice"
	"github.com/fatih/structs"
)

const (
	// TorrentStatusNormal Int for Torrent status normal
	TorrentStatusNormal = 1
	// TorrentStatusRemake Int for Torrent status remake
	TorrentStatusRemake = 2
	// TorrentStatusTrusted Int for Torrent status trusted
	TorrentStatusTrusted = 3
	// TorrentStatusAPlus Int for Torrent status a+
	TorrentStatusAPlus = 4
	// TorrentStatusBlocked Int for Torrent status locked
	TorrentStatusBlocked = 5
)

// Torrent model
type Torrent struct {
	ID          uint      `gorm:"column:torrent_id;primary_key"`
	Name        string    `gorm:"column:torrent_name"`
	Hash        string    `gorm:"column:torrent_hash;unique"`
	Category    int       `gorm:"column:category"`
	SubCategory int       `gorm:"column:sub_category"`
	Status      int       `gorm:"column:status"`
	Hidden      bool      `gorm:"column:hidden"`
	Date        time.Time `gorm:"column:date"`
	UploaderID  uint      `gorm:"column:uploader"`
	Stardom     int       `gorm:"column:stardom"`
	Filesize    int64     `gorm:"column:filesize"`
	Description string    `gorm:"column:description"`
	WebsiteLink string    `gorm:"column:website_link"`
	Trackers    string    `gorm:"column:trackers"`

	// Torrent Details
	AnidbID      uint   `gorm:"column:anidbid"`
	VndbID       uint   `gorm:"column:vndbid"`
	VgmdbID      uint   `gorm:"column:vgmdbid"`
	Dlsite       string `gorm:"column:dlsite"`
	VideoQuality string `gorm:"column:videoquality"`
	AcceptedTags string `gorm:"column:tags"`
	// Indicates the language of the torrent's content (eg. subs, dubs, raws, manga TLs)
	Language  string `gorm:"column:language"`
	DeletedAt *time.Time

	Uploader    *User        `gorm:"AssociationForeignKey:UploaderID;ForeignKey:user_id"`
	OldUploader string       `gorm:"-"` // ???????
	OldComments []OldComment `gorm:"ForeignKey:torrent_id"`
	Comments    []Comment    `gorm:"ForeignKey:torrent_id"`
	Tags        Tags         `gorm:"-"`
	Scrape      *Scrape      `gorm:"AssociationForeignKey:ID;ForeignKey:torrent_id"`
	FileList    []File       `gorm:"ForeignKey:torrent_id"`
	Languages   []string     `gorm:"-"` // This is parsed when retrieved from db
}

/* We need a JSON object instead of a Gorm structure because magnet URLs are
   not in the database and have to be generated dynamically */

// TorrentJSON for torrent model in json for api
type TorrentJSON struct {
	ID          uint          `json:"id"`
	Name        string        `json:"name"`
	Status      int           `json:"status"`
	Hidden      bool          `json:"-"`
	Hash        string        `json:"hash"`
	Date        string        `json:"date"`
	FullDate    time.Time     `json:"-"` //Used to convert the date to full OR short format depending on the situation
	Filesize    int64         `json:"filesize"`
	Description template.HTML `json:"description"`
	Comments    []CommentJSON `json:"comments"`
	SubCategory string        `json:"sub_category"`
	Category    string        `json:"category"`

	// Torrent DBID
	AnidbID      uint   `json:"anidbid"`
	VndbID       uint   `json:"vndbid"`
	VgmdbID      uint   `json:"vgmdbid"`
	Dlsite       string `json:"dlsite"`
	VideoQuality string `json:"videoquality"`
	AcceptedTags Tags   `json:"tags"`

	UploaderID    uint          `json:"uploader_id"`
	UploaderName  template.HTML `json:"uploader_name"`
	OldUploader   template.HTML `json:"uploader_old"`
	WebsiteLink   template.URL  `json:"website_link"`
	Languages     []string      `json:"languages"`
	Magnet        template.URL  `json:"magnet"`
	TorrentLink   template.URL  `json:"torrent"`
	Seeders       uint32        `json:"seeders"`
	Leechers      uint32        `json:"leechers"`
	Completed     uint32        `json:"completed"`
	LastScrape    time.Time     `json:"last_scrape"`
	StatsObsolete []bool        `json:"-"` //First cell determines whether the stats are valid, second determines whether the stats need a refresh regardless of first cell (too old stats?)
	FileList      []FileJSON    `json:"file_list"`
	Tags          Tags          `json:"-"` // not needed in json to reduce db calls
}

// Size : Returns the total size of memory recursively allocated for this struct
// FIXME: Is it deprecated?
func (t Torrent) Size() (s int) {
	s = int(reflect.TypeOf(t).Size())
	return
}

// TableName : Return the table name of torrents table
func (t Torrent) TableName() string {
	return config.Get().Models.TorrentsTableName
}

// Identifier : Return the identifier of a torrent
func (t *Torrent) Identifier() string {
	return fmt.Sprintf("torrent_%d", t.ID)
}

// IsNormal : Return if a torrent status is normal
func (t *Torrent) IsNormal() bool {
	return t.Status == TorrentStatusNormal
}

// IsRemake : Return if a torrent status is remake
func (t *Torrent) IsRemake() bool {
	return t.Status == TorrentStatusRemake
}

// IsTrusted : Return if a torrent status is trusted
func (t *Torrent) IsTrusted() bool {
	return t.Status == TorrentStatusTrusted
}

// IsAPlus : Return if a torrent status is a+
func (t *Torrent) IsAPlus() bool {
	return t.Status == TorrentStatusAPlus
}

// IsBlocked : Return if a torrent status is locked
func (t *Torrent) IsBlocked() bool {
	return t.Status == TorrentStatusBlocked
}

// IsDeleted : Return if a torrent status is deleted
func (t *Torrent) IsDeleted() bool {
	return t.DeletedAt != nil
}

// GetDescriptiveTags : Return the descriptive tags
func (t *Torrent) GetDescriptiveTags() string {
	return t.AcceptedTags
}

// AddToESIndex : Adds a torrent to Elastic Search
func (t Torrent) AddToESIndex(client *elastic.Client) error {
	ctx := context.Background()
	torrentJSON := t.ToJSON()
	_, err := client.Index().
		Index(config.Get().Search.ElasticsearchIndex).
		Type(config.Get().Search.ElasticsearchType).
		Id(strconv.FormatUint(uint64(torrentJSON.ID), 10)).
		BodyJson(torrentJSON).
		Refresh("true").
		Do(ctx)
	return err
}

// DeleteFromESIndex : Removes a torrent from Elastic Search
func (t *Torrent) DeleteFromESIndex(client *elastic.Client) error {
	ctx := context.Background()
	_, err := client.Delete().
		Index(config.Get().Search.ElasticsearchIndex).
		Type(config.Get().Search.ElasticsearchType).
		Id(strconv.FormatInt(int64(t.ID), 10)).
		Do(ctx)
	return err
}

// ParseTrackers : Takes an array of trackers, adds needed trackers and parse it to url string
func (t *Torrent) ParseTrackers(trackers []string) {
	v := url.Values{}
	if len(config.Get().Torrents.Trackers.NeededTrackers) > 0 { // if we have some needed trackers configured
		if len(trackers) == 0 {
			trackers = config.Get().Torrents.Trackers.Default
		} else {
			for _, id := range config.Get().Torrents.Trackers.NeededTrackers {
				found := false
				for _, tracker := range trackers {
					if tracker == config.Get().Torrents.Trackers.Default[id] {
						found = true
						break
					}
				}
				if !found {
					trackers = append(trackers, config.Get().Torrents.Trackers.Default[id])
				}
			}
		}
	}
	v["tr"] = trackers
	t.Trackers = v.Encode()
}

func (t *Torrent) ParseLanguages() {
	t.Languages = strings.Split(t.Language, ",")
}

func (t *Torrent) EncodeLanguages() {
	t.Language = strings.Join(t.Languages, ",")
}

// GetTrackersArray : Convert trackers string to Array
func (t *Torrent) GetTrackersArray() (trackers []string) {
	v, _ := url.ParseQuery(t.Trackers)
	trackers = v["tr"]
	return
}

// ToTorrent :
// TODO: Need to get rid of TorrentJSON altogether and have only one true Torrent
//       model
func (t *TorrentJSON) ToTorrent() Torrent {
	category, err := strconv.ParseInt(t.Category, 10, 64)
	if err != nil {
		category = 0
	}
	subCategory, err := strconv.ParseInt(t.SubCategory, 10, 64)
	if err != nil {
		subCategory = 0
	}
	// Need to add +00:00 at the end because ES doesn't store it by default
	dateFixed := t.Date
	if len(dateFixed) > 6 && dateFixed[len(dateFixed)-6] != '+' {
		dateFixed += "Z"
	}
	date, err := time.Parse(time.RFC3339, dateFixed)
	if err != nil {
		log.Errorf("Problem parsing date '%s' from ES: %s", dateFixed, err)
	}
	torrent := Torrent{
		ID:          t.ID,
		Name:        t.Name,
		Hash:        t.Hash,
		Category:    int(category),
		SubCategory: int(subCategory),
		Status:      t.Status,
		Date:        date,
		UploaderID:  t.UploaderID,
		AnidbID:     t.AnidbID,
		VndbID:      t.VndbID,
		VgmdbID:     t.VgmdbID,
		Dlsite:      t.Dlsite,
		//Stardom: t.Stardom,
		Filesize:    t.Filesize,
		Description: string(t.Description),
		Hidden:      t.Hidden,
		//WebsiteLink: t.WebsiteLink,
		//Trackers: t.Trackers,
		//DeletedAt: t.DeletedAt,
		// Uploader: TODO
		//OldUploader: t.OldUploader,
		//OldComments: TODO
		// Comments: TODO
		// LastScrape not stored in ES, counts won't show without a value however
		Scrape:    &Scrape{Seeders: t.Seeders, Leechers: t.Leechers, Completed: t.Completed, LastScrape: time.Now()},
		Languages: t.Languages,
		//FileList: TODO
	}
	torrent.EncodeLanguages()
	return torrent
}

// ToJSON converts a models.Torrent to its equivalent JSON structure
func (t *Torrent) ToJSON() TorrentJSON {
	var trackers []string
	if t.Trackers == "" {
		trackers = config.Get().Torrents.Trackers.Default
	} else {
		trackers = t.GetTrackersArray()
	}
	magnet := format.InfoHashToMagnet(strings.TrimSpace(t.Hash), t.Name, trackers...)
	commentsJSON := make([]CommentJSON, 0, len(t.OldComments)+len(t.Comments))
	for _, c := range t.OldComments {
		commentsJSON = append(commentsJSON, CommentJSON{Username: c.Username, UserID: -1, UserStatus: "", Content: template.HTML(c.Content), Date: c.Date.UTC()})
	}
	for _, c := range t.Comments {
		if c.User != nil {
			userStatus := ""
			if c.User.IsBanned() {
				userStatus = "userstatus_banned"
			}
			if c.User.HasAdmin() {
				userStatus =  "userstatus_moderator"
			}
			if c.User.ID == t.ID {
				userStatus = "userstatus_uploader"
			}
			commentsJSON = append(commentsJSON, CommentJSON{Username: c.User.Username, UserID: int(c.User.ID), UserStatus: userStatus, Content: sanitize.MarkdownToHTML(c.Content), Date: c.CreatedAt.UTC(), UserAvatar: c.User.MD5})
		} else {
			commentsJSON = append(commentsJSON, CommentJSON{})
		}
	}

	// Sort comments by date
	slice.Sort(commentsJSON, func(i, j int) bool {
		return commentsJSON[i].Date.Before(commentsJSON[j].Date)
	})

	fileListJSON := make([]FileJSON, 0, len(t.FileList))
	for _, f := range t.FileList {
		fileListJSON = append(fileListJSON, FileJSON{
			Path:     filepath.Join(f.Path()...),
			Filesize: f.Filesize,
		})
	}

	// Sort file list by lowercase filename
	slice.Sort(fileListJSON, func(i, j int) bool {
		return strings.ToLower(fileListJSON[i].Path) < strings.ToLower(fileListJSON[j].Path)
	})

	uploader := "れんちょん" // by default
	var uploaderID uint
	if t.UploaderID > 0 && t.Uploader != nil {
		uploader = t.Uploader.Username
		uploaderID = t.UploaderID
	} else if t.OldUploader != "" {
		uploader = t.OldUploader
	}
	torrentlink := ""
	if len(config.Get().Torrents.CacheLink) > 0 { // Only use torrent cache if set, don't check id since better to have all .torrent
		if config.IsSukebei() {
			torrentlink = "" // torrent cache doesn't have sukebei torrents
		} else {
			torrentlink = fmt.Sprintf(config.Get().Torrents.CacheLink, t.Hash)
		}
	} else if len(config.Get().Torrents.StorageLink) > 0 { // Only use own .torrent if storage set
		torrentlink = fmt.Sprintf(config.Get().Torrents.StorageLink, t.Hash)
	}
	scrape := Scrape{}
	if t.Scrape != nil {
		scrape = *t.Scrape
	}
	
	statsObsolete := []bool{false, false}
	
	if scrape.LastScrape.IsZero() || (scrape.Seeders == 0 && scrape.Leechers == 0 && scrape.Completed == 0) {
		statsObsolete[0] = true
		//The displayed stats are obsolete, S/D/L will show "Unknown"
	}
	if time.Since(scrape.LastScrape).Hours() > config.Get().Scrape.StatScrapingFrequency || (scrape.Seeders == 0 && scrape.Leechers == 0 && scrape.Completed == 0 && time.Since(scrape.LastScrape).Hours() >= config.Get().Scrape.StatScrapingFrequencyUnknown) {
		statsObsolete[1] = true
		//The stats need to be refreshed, either because they are valid and older than one month (not that reliable) OR if they are unknown but have been scraped 1h (or more) ago
	}
	
	t.ParseLanguages()
	res := TorrentJSON{
		ID:            t.ID,
		Name:          t.Name,
		Status:        t.Status,
		Hidden:        t.Hidden,
		Hash:          t.Hash,
		Date:          t.Date.UTC().Format(time.RFC3339),
		FullDate:      t.Date.UTC(),
		Filesize:      t.Filesize,
		Description:   sanitize.MarkdownToHTML(t.Description),
		Comments:      commentsJSON,
		SubCategory:   strconv.Itoa(t.SubCategory),
		Category:      strconv.Itoa(t.Category),
		UploaderID:    uploaderID,
		UploaderName:  sanitize.SafeText(uploader),
		WebsiteLink:   sanitize.Safe(t.WebsiteLink),
		Languages:     t.Languages,
		Magnet:        template.URL(magnet),
		TorrentLink:   sanitize.Safe(torrentlink),
		Leechers:      scrape.Leechers,
		Seeders:       scrape.Seeders,
		Completed:     scrape.Completed,
		LastScrape:    scrape.LastScrape,
		StatsObsolete: statsObsolete,
		FileList:      fileListJSON,
		Tags:          t.Tags,
		AnidbID:       t.AnidbID,
		VndbID:        t.VndbID,
		VgmdbID:       t.VgmdbID,
		Dlsite:        t.Dlsite,
		VideoQuality:  t.VideoQuality,
	}

	// Split accepted tags
	tags := strings.Split(t.AcceptedTags, ",")
	for _, tag := range tags {
		if tag != "" {
			res.AcceptedTags = append(res.AcceptedTags, Tag{Tag: tag, Type: config.Get().Torrents.Tags.Default, Total: config.Get().Torrents.Tags.MaxWeight, Accepted: true})
		}
	}

	return res
}

/* Complete the functions when necessary... */

// TorrentsToJSON : Map Torrents to TorrentsToJSON without reallocations
func TorrentsToJSON(t []Torrent) []TorrentJSON {
	json := make([]TorrentJSON, len(t))
	for i := range t {
		json[i] = t[i].ToJSON()
	}
	return json
}

// Update : Update a torrent based on model
func (t *Torrent) Update(unscope bool) (int, error) {
	db := ORM
	if unscope {
		db = ORM.Unscoped()
	}
	t.EncodeLanguages() // Need to transform array into single string

	if db.Model(t).UpdateColumn(t.toMap()).Error != nil {
		return http.StatusInternalServerError, errors.New("Torrent was not updated")
	}

	// TODO Don't create a new client for each request
	if ElasticSearchClient != nil {
		err := t.AddToESIndex(ElasticSearchClient)
		if err == nil {
			log.Infof("Successfully updated torrent to ES index.")
		} else {
			log.Errorf("Unable to update torrent to ES index: %s", err)
		}
	}
	// We only flush cache after update
	cache.C.Delete(t.Identifier())

	return http.StatusOK, nil
}

// UpdateUnscope : Update a torrent based on model
func (t *Torrent) UpdateUnscope() (int, error) {
	return t.Update(true)
}

// Delete : delete a torrent based on id
func (t *Torrent) Delete(definitely bool) (*Torrent, int, error) {
	if t.ID == 0 {
		err := errors.New("ERROR: Tried to delete a torrent with ID 0")
		log.CheckErrorWithMessage(err, "ERROR_IMPORTANT: ")
		return t, http.StatusBadRequest, err
	}
	db := ORM
	if definitely {
		db = ORM.Unscoped()
	}
	if db.Delete(t).Error != nil {
		return t, http.StatusInternalServerError, errors.New("torrent_not_deleted")
	}

	if ElasticSearchClient != nil {
		err := t.DeleteFromESIndex(ElasticSearchClient)
		if err == nil {
			log.Infof("Successfully deleted torrent to ES index.")
		} else {
			log.Errorf("Unable to delete torrent to ES index: %s", err)
		}
	}
	// We flush cache only after delete
	cache.C.Flush()
	return t, http.StatusOK, nil
}

// DefinitelyDelete : deletes definitely a torrent based on id
func (t *Torrent) DefinitelyDelete() (*Torrent, int, error) {
	return t.Delete(true)

}

// toMap : convert the model to a map of interface
func (t *Torrent) toMap() map[string]interface{} {
	return structs.Map(t)
}

// LoadTags : load all the unique tags with summed up weight from the database in torrent
func (t *Torrent) LoadTags() {
	// Only load if necessary
	if len(t.Tags) == 0 {
		// Should output a query like this: SELECT tag, type, accepted, SUM(weight) as total FROM tags WHERE torrent_id=923000 GROUP BY type, tag ORDER BY type, total DESC
		err := ORM.Select("tag, type, SUM(weight) as total").Where("torrent_id = ?", t.ID).Group("type, tag").Order("type ASC, total DESC").Find(&t.Tags).Error
		log.CheckErrorWithMessage(err, "LOAD_TAGS_ERROR: Couldn't load tags from DB!")
	}
}

// DeleteTags cipes out all the tags from the torrent. Doesn't decrease pantsu on users!
func (t *Torrent) DeleteTags() {
	if t.ID > 0 {
		// Should output a query like this: DELETE FROM tags WHERE torrent_id=923000
		err := ORM.Where("torrent_id = ?", t.ID).Delete(&t.Tags).Error
		log.CheckErrorWithMessage(err, "LOAD_TAGS_ERROR: Couldn't delete tags!")
	}
}