package models import ( "context" "errors" "fmt" "html/template" "path/filepath" "reflect" "strconv" "strings" "time" elastic "gopkg.in/olivere/elastic.v5" "net/http" "net/url" "github.com/NyaaPantsu/nyaa/config" "github.com/NyaaPantsu/nyaa/utils/cache" "github.com/NyaaPantsu/nyaa/utils/format" "github.com/NyaaPantsu/nyaa/utils/log" "github.com/NyaaPantsu/nyaa/utils/sanitize" "github.com/bradfitz/slice" "github.com/fatih/structs" ) const ( // TorrentStatusNormal Int for Torrent status normal TorrentStatusNormal = 1 // TorrentStatusRemake Int for Torrent status remake TorrentStatusRemake = 2 // TorrentStatusTrusted Int for Torrent status trusted TorrentStatusTrusted = 3 // TorrentStatusAPlus Int for Torrent status a+ TorrentStatusAPlus = 4 // TorrentStatusBlocked Int for Torrent status locked TorrentStatusBlocked = 5 ) // Torrent model type Torrent struct { ID uint `gorm:"column:torrent_id;primary_key"` Name string `gorm:"column:torrent_name"` Hash string `gorm:"column:torrent_hash;unique"` Category int `gorm:"column:category"` SubCategory int `gorm:"column:sub_category"` Status int `gorm:"column:status"` Hidden bool `gorm:"column:hidden"` Date time.Time `gorm:"column:date"` UploaderID uint `gorm:"column:uploader"` Stardom int `gorm:"column:stardom"` Filesize int64 `gorm:"column:filesize"` Description string `gorm:"column:description"` WebsiteLink string `gorm:"column:website_link"` Trackers string `gorm:"column:trackers"` // Torrent Details AnidbID uint `gorm:"column:anidbid"` VndbID uint `gorm:"column:vndbid"` VgmdbID uint `gorm:"column:vgmdbid"` Dlsite string `gorm:"column:dlsite"` VideoQuality string `gorm:"column:videoquality"` AcceptedTags string `gorm:"column:tags"` // Indicates the language of the torrent's content (eg. subs, dubs, raws, manga TLs) Language string `gorm:"column:language"` DeletedAt *time.Time Uploader *User `gorm:"AssociationForeignKey:UploaderID;ForeignKey:user_id"` OldUploader string `gorm:"-"` // ??????? OldComments []OldComment `gorm:"ForeignKey:torrent_id"` Comments []Comment `gorm:"ForeignKey:torrent_id"` Tags Tags `gorm:"-"` Scrape *Scrape `gorm:"AssociationForeignKey:ID;ForeignKey:torrent_id"` FileList []File `gorm:"ForeignKey:torrent_id"` Languages []string `gorm:"-"` // This is parsed when retrieved from db } /* We need a JSON object instead of a Gorm structure because magnet URLs are not in the database and have to be generated dynamically */ // TorrentJSON for torrent model in json for api type TorrentJSON struct { ID uint `json:"id"` Name string `json:"name"` Status int `json:"status"` Hidden bool `json:"-"` Hash string `json:"hash"` Date string `json:"date"` FullDate time.Time `json:"-"` //Used to convert the date to full OR short format depending on the situation Filesize int64 `json:"filesize"` Description template.HTML `json:"description"` Comments []CommentJSON `json:"comments"` SubCategory string `json:"sub_category"` Category string `json:"category"` // Torrent DBID AnidbID uint `json:"anidbid"` VndbID uint `json:"vndbid"` VgmdbID uint `json:"vgmdbid"` Dlsite string `json:"dlsite"` VideoQuality string `json:"videoquality"` AcceptedTags Tags `json:"tags"` UploaderID uint `json:"uploader_id"` UploaderName template.HTML `json:"uploader_name"` OldUploader template.HTML `json:"uploader_old"` WebsiteLink template.URL `json:"website_link"` Languages []string `json:"languages"` Magnet template.URL `json:"magnet"` TorrentLink template.URL `json:"torrent"` Seeders uint32 `json:"seeders"` Leechers uint32 `json:"leechers"` Completed uint32 `json:"completed"` LastScrape time.Time `json:"last_scrape"` StatsObsolete []bool `json:"-"` //First cell determines whether the stats are valid, second determines whether the stats need a refresh regardless of first cell (too old stats?) FileList []FileJSON `json:"file_list"` Tags Tags `json:"-"` // not needed in json to reduce db calls } // Size : Returns the total size of memory recursively allocated for this struct // FIXME: Is it deprecated? func (t Torrent) Size() (s int) { s = int(reflect.TypeOf(t).Size()) return } // TableName : Return the table name of torrents table func (t Torrent) TableName() string { return config.Get().Models.TorrentsTableName } // Identifier : Return the identifier of a torrent func (t *Torrent) Identifier() string { return fmt.Sprintf("torrent_%d", t.ID) } // IsNormal : Return if a torrent status is normal func (t *Torrent) IsNormal() bool { return t.Status == TorrentStatusNormal } // IsRemake : Return if a torrent status is remake func (t *Torrent) IsRemake() bool { return t.Status == TorrentStatusRemake } // IsTrusted : Return if a torrent status is trusted func (t *Torrent) IsTrusted() bool { return t.Status == TorrentStatusTrusted } // IsAPlus : Return if a torrent status is a+ func (t *Torrent) IsAPlus() bool { return t.Status == TorrentStatusAPlus } // IsBlocked : Return if a torrent status is locked func (t *Torrent) IsBlocked() bool { return t.Status == TorrentStatusBlocked } // IsDeleted : Return if a torrent status is deleted func (t *Torrent) IsDeleted() bool { return t.DeletedAt != nil } // GetDescriptiveTags : Return the descriptive tags func (t *Torrent) GetDescriptiveTags() string { return t.AcceptedTags } // AddToESIndex : Adds a torrent to Elastic Search func (t Torrent) AddToESIndex(client *elastic.Client) error { ctx := context.Background() torrentJSON := t.ToJSON() _, err := client.Index(). Index(config.Get().Search.ElasticsearchIndex). Type(config.Get().Search.ElasticsearchType). Id(strconv.FormatUint(uint64(torrentJSON.ID), 10)). BodyJson(torrentJSON). Refresh("true"). Do(ctx) return err } // DeleteFromESIndex : Removes a torrent from Elastic Search func (t *Torrent) DeleteFromESIndex(client *elastic.Client) error { ctx := context.Background() _, err := client.Delete(). Index(config.Get().Search.ElasticsearchIndex). Type(config.Get().Search.ElasticsearchType). Id(strconv.FormatInt(int64(t.ID), 10)). Do(ctx) return err } // ParseTrackers : Takes an array of trackers, adds needed trackers and parse it to url string func (t *Torrent) ParseTrackers(trackers []string) { v := url.Values{} if len(config.Get().Torrents.Trackers.NeededTrackers) > 0 { // if we have some needed trackers configured if len(trackers) == 0 { trackers = config.Get().Torrents.Trackers.Default } else { for _, id := range config.Get().Torrents.Trackers.NeededTrackers { found := false for _, tracker := range trackers { if tracker == config.Get().Torrents.Trackers.Default[id] { found = true break } } if !found { trackers = append(trackers, config.Get().Torrents.Trackers.Default[id]) } } } } v["tr"] = trackers t.Trackers = v.Encode() } func (t *Torrent) ParseLanguages() { t.Languages = strings.Split(t.Language, ",") } func (t *Torrent) EncodeLanguages() { t.Language = strings.Join(t.Languages, ",") } // GetTrackersArray : Convert trackers string to Array func (t *Torrent) GetTrackersArray() (trackers []string) { v, _ := url.ParseQuery(t.Trackers) trackers = v["tr"] return } // ToTorrent : // TODO: Need to get rid of TorrentJSON altogether and have only one true Torrent // model func (t *TorrentJSON) ToTorrent() Torrent { category, err := strconv.ParseInt(t.Category, 10, 64) if err != nil { category = 0 } subCategory, err := strconv.ParseInt(t.SubCategory, 10, 64) if err != nil { subCategory = 0 } // Need to add +00:00 at the end because ES doesn't store it by default dateFixed := t.Date if len(dateFixed) > 6 && dateFixed[len(dateFixed)-6] != '+' { dateFixed += "Z" } date, err := time.Parse(time.RFC3339, dateFixed) if err != nil { log.Errorf("Problem parsing date '%s' from ES: %s", dateFixed, err) } torrent := Torrent{ ID: t.ID, Name: t.Name, Hash: t.Hash, Category: int(category), SubCategory: int(subCategory), Status: t.Status, Date: date, UploaderID: t.UploaderID, AnidbID: t.AnidbID, VndbID: t.VndbID, VgmdbID: t.VgmdbID, Dlsite: t.Dlsite, //Stardom: t.Stardom, Filesize: t.Filesize, Description: string(t.Description), Hidden: t.Hidden, //WebsiteLink: t.WebsiteLink, //Trackers: t.Trackers, //DeletedAt: t.DeletedAt, // Uploader: TODO //OldUploader: t.OldUploader, //OldComments: TODO // Comments: TODO // LastScrape not stored in ES, counts won't show without a value however Scrape: &Scrape{Seeders: t.Seeders, Leechers: t.Leechers, Completed: t.Completed, LastScrape: time.Now()}, Languages: t.Languages, //FileList: TODO } torrent.EncodeLanguages() return torrent } // ToJSON converts a models.Torrent to its equivalent JSON structure func (t *Torrent) ToJSON() TorrentJSON { var trackers []string if t.Trackers == "" { trackers = config.Get().Torrents.Trackers.Default } else { trackers = t.GetTrackersArray() } magnet := format.InfoHashToMagnet(strings.TrimSpace(t.Hash), t.Name, trackers...) commentsJSON := make([]CommentJSON, 0, len(t.OldComments)+len(t.Comments)) for _, c := range t.OldComments { commentsJSON = append(commentsJSON, CommentJSON{Username: c.Username, UserID: -1, Content: template.HTML(c.Content), Date: c.Date.UTC()}) } for _, c := range t.Comments { if c.User != nil { commentsJSON = append(commentsJSON, CommentJSON{Username: c.User.Username, UserID: int(c.User.ID), Content: sanitize.MarkdownToHTML(c.Content), Date: c.CreatedAt.UTC(), UserAvatar: c.User.MD5}) } else { commentsJSON = append(commentsJSON, CommentJSON{}) } } // Sort comments by date slice.Sort(commentsJSON, func(i, j int) bool { return commentsJSON[i].Date.Before(commentsJSON[j].Date) }) fileListJSON := make([]FileJSON, 0, len(t.FileList)) for _, f := range t.FileList { fileListJSON = append(fileListJSON, FileJSON{ Path: filepath.Join(f.Path()...), Filesize: f.Filesize, }) } // Sort file list by lowercase filename slice.Sort(fileListJSON, func(i, j int) bool { return strings.ToLower(fileListJSON[i].Path) < strings.ToLower(fileListJSON[j].Path) }) uploader := "れんちょん" // by default var uploaderID uint if t.UploaderID > 0 && t.Uploader != nil { uploader = t.Uploader.Username uploaderID = t.UploaderID } else if t.OldUploader != "" { uploader = t.OldUploader } torrentlink := "" if t.ID <= config.Get().Models.LastOldTorrentID && len(config.Get().Torrents.CacheLink) > 0 { if config.IsSukebei() { torrentlink = "" // torrent cache doesn't have sukebei torrents } else { torrentlink = fmt.Sprintf(config.Get().Torrents.CacheLink, t.Hash) } } else if t.ID > config.Get().Models.LastOldTorrentID && len(config.Get().Torrents.StorageLink) > 0 { torrentlink = fmt.Sprintf(config.Get().Torrents.StorageLink, t.Hash) } scrape := Scrape{} if t.Scrape != nil { scrape = *t.Scrape } statsObsolete := []bool{false, false} if scrape.LastScrape.IsZero() || (scrape.Seeders == 0 && scrape.Leechers == 0 && scrape.Completed == 0) { statsObsolete[0] = true //The displayed stats are obsolete, S/D/L will show "Unknown" } if time.Since(scrape.LastScrape).Hours() > 730 || (scrape.Seeders == 0 && scrape.Leechers == 0 && scrape.Completed == 0 && time.Since(scrape.LastScrape).Hours() > 12) { statsObsolete[1] = true //The stats need to be refreshed, either because they are valid and older than one months (not as reliable) OR if they are unknown but have been scraped more than 12h ago } t.ParseLanguages() res := TorrentJSON{ ID: t.ID, Name: t.Name, Status: t.Status, Hidden: t.Hidden, Hash: t.Hash, Date: t.Date.Format(time.RFC3339), FullDate: t.Date, Filesize: t.Filesize, Description: sanitize.MarkdownToHTML(t.Description), Comments: commentsJSON, SubCategory: strconv.Itoa(t.SubCategory), Category: strconv.Itoa(t.Category), UploaderID: uploaderID, UploaderName: sanitize.SafeText(uploader), WebsiteLink: sanitize.Safe(t.WebsiteLink), Languages: t.Languages, Magnet: template.URL(magnet), TorrentLink: sanitize.Safe(torrentlink), Leechers: scrape.Leechers, Seeders: scrape.Seeders, Completed: scrape.Completed, LastScrape: scrape.LastScrape, StatsObsolete: statsObsolete, FileList: fileListJSON, Tags: t.Tags, AnidbID: t.AnidbID, VndbID: t.VndbID, VgmdbID: t.VgmdbID, Dlsite: t.Dlsite, VideoQuality: t.VideoQuality, } // Split accepted tags tags := strings.Split(t.AcceptedTags, ",") for _, tag := range tags { if tag != "" { res.AcceptedTags = append(res.AcceptedTags, Tag{Tag: tag, Type: config.Get().Torrents.Tags.Default, Total: config.Get().Torrents.Tags.MaxWeight, Accepted: true}) } } return res } /* Complete the functions when necessary... */ // TorrentsToJSON : Map Torrents to TorrentsToJSON without reallocations func TorrentsToJSON(t []Torrent) []TorrentJSON { json := make([]TorrentJSON, len(t)) for i := range t { json[i] = t[i].ToJSON() } return json } // Update : Update a torrent based on model func (t *Torrent) Update(unscope bool) (int, error) { db := ORM if unscope { db = ORM.Unscoped() } t.EncodeLanguages() // Need to transform array into single string if db.Model(t).UpdateColumn(t.toMap()).Error != nil { return http.StatusInternalServerError, errors.New("Torrent was not updated") } // TODO Don't create a new client for each request if ElasticSearchClient != nil { err := t.AddToESIndex(ElasticSearchClient) if err == nil { log.Infof("Successfully updated torrent to ES index.") } else { log.Errorf("Unable to update torrent to ES index: %s", err) } } // We only flush cache after update cache.C.Delete(t.Identifier()) return http.StatusOK, nil } // UpdateUnscope : Update a torrent based on model func (t *Torrent) UpdateUnscope() (int, error) { return t.Update(true) } // Delete : delete a torrent based on id func (t *Torrent) Delete(definitely bool) (*Torrent, int, error) { if t.ID == 0 { err := errors.New("ERROR: Tried to delete a torrent with ID 0") log.CheckErrorWithMessage(err, "ERROR_IMPORTANT: ") return t, http.StatusBadRequest, err } db := ORM if definitely { db = ORM.Unscoped() } if db.Delete(t).Error != nil { return t, http.StatusInternalServerError, errors.New("torrent_not_deleted") } if ElasticSearchClient != nil { err := t.DeleteFromESIndex(ElasticSearchClient) if err == nil { log.Infof("Successfully deleted torrent to ES index.") } else { log.Errorf("Unable to delete torrent to ES index: %s", err) } } // We flush cache only after delete cache.C.Flush() return t, http.StatusOK, nil } // DefinitelyDelete : deletes definitely a torrent based on id func (t *Torrent) DefinitelyDelete() (*Torrent, int, error) { return t.Delete(true) } // toMap : convert the model to a map of interface func (t *Torrent) toMap() map[string]interface{} { return structs.Map(t) } // LoadTags : load all the unique tags with summed up weight from the database in torrent func (t *Torrent) LoadTags() { // Only load if necessary if len(t.Tags) == 0 { // Should output a query like this: SELECT tag, type, accepted, SUM(weight) as total FROM tags WHERE torrent_id=923000 GROUP BY type, tag ORDER BY type, total DESC err := ORM.Select("tag, type, SUM(weight) as total").Where("torrent_id = ?", t.ID).Group("type, tag").Order("type ASC, total DESC").Find(&t.Tags).Error log.CheckErrorWithMessage(err, "LOAD_TAGS_ERROR: Couldn't load tags from DB!") } } // DeleteTags cipes out all the tags from the torrent. Doesn't decrease pantsu on users! func (t *Torrent) DeleteTags() { if t.ID > 0 { // Should output a query like this: DELETE FROM tags WHERE torrent_id=923000 err := ORM.Where("torrent_id = ?", t.ID).Delete(&t.Tags).Error log.CheckErrorWithMessage(err, "LOAD_TAGS_ERROR: Couldn't delete tags!") } }