package search import ( "encoding/base64" "encoding/json" "fmt" "strconv" "strings" "unicode" "unicode/utf8" elastic "gopkg.in/olivere/elastic.v5" "github.com/NyaaPantsu/nyaa/models" "github.com/NyaaPantsu/nyaa/models/torrents" "github.com/NyaaPantsu/nyaa/utils/log" "github.com/NyaaPantsu/nyaa/utils/publicSettings" "github.com/gin-gonic/gin" ) // TorrentParam defines all parameters that can be provided when searching for a torrent type TorrentParam struct { Full bool // True means load all members Order bool // True means ascending Hidden bool // True means filter hidden torrents Deleted bool // False means filter deleted torrents Status Status Sort SortMode Category Categories Max maxType Offset uint32 UserID uint32 TorrentID []uint32 FromID uint32 FromDate DateFilter ToDate DateFilter NotNull string // csv NameLike string // csv Languages publicSettings.Languages MinSize SizeBytes MaxSize SizeBytes // Tags search AnidbID uint32 VndbID uint32 VgmdbID uint32 Dlsite uint32 VideoQuality string Tags Tags } // Identifier returns a unique identifier for the struct func (p *TorrentParam) Identifier() string { cats := "" for _, v := range p.Category { cats += fmt.Sprintf("%d%d", v.Main, v.Sub) } languages := "" for _, v := range p.Languages { languages += fmt.Sprintf("%s%s", v.Code, v.Name) } ids := "" for _, v := range p.TorrentID { ids += fmt.Sprintf("%d", v) } // Tags identifier tags := strings.Join(p.Tags, ",") tags += p.VideoQuality dbids := fmt.Sprintf("%d%d%d%d", p.AnidbID, p.VndbID, p.VgmdbID, p.Dlsite) identifier := fmt.Sprintf("%s%s%s%d%d%d%d%d%d%d%s%s%s%d%s%s%s%t%t%t%t", p.NameLike, p.NotNull, languages, p.Max, p.Offset, p.FromID, p.MinSize, p.MaxSize, p.Status, p.Sort, dbids, p.FromDate, p.ToDate, p.UserID, ids, cats, tags, p.Full, p.Order, p.Hidden, p.Deleted) return base64.URLEncoding.EncodeToString([]byte(identifier)) } func parseUInt(c *gin.Context, key string) uint32 { // Get the user id from the url u64, err := strconv.ParseUint(c.Query(key), 10, 32) if err != nil { // if you can't convert it, you set it to 0 u64 = 0 } return uint32(u64) } func parseTorrentID(c *gin.Context) (uint32, []uint32) { // Get the torrent ID to limit the results to the ones after this torrent fromID, err := strconv.ParseUint(c.Query("fromID"), 10, 32) if err != nil { // if you can't convert it, you set it to 0 fromID = 0 } var torrentIDs []uint32 ids := c.QueryArray("id") for _, id := range ids { idInt, err := strconv.Atoi(id) if err == nil { torrentIDs = append(torrentIDs, uint32(idInt)) } } return uint32(fromID), torrentIDs } func parseOrder(c *gin.Context) bool { if c.Query("order") == "true" { return true } return false } // FromRequest : parse a request in torrent param // TODO Should probably return an error ? func (p *TorrentParam) FromRequest(c *gin.Context) { // Search by name // We take the search arguments from "q" in url p.NameLike = strings.TrimSpace(c.Query("q")) // Maximum results returned // We take the maxximum results to display from "limit" in url p.Max.Parse(c.Query("limit")) // Limit search to one user // Get the user id from the url p.UserID = parseUInt(c, "userID") // Limit search to DbID // Get the id from the url p.AnidbID = parseUInt(c, "anidb") p.VndbID = parseUInt(c, "vndb") p.VgmdbID = parseUInt(c, "vgm") p.Dlsite = parseUInt(c, "dlsite") // Limit search to video quality // Get the video quality from url p.VideoQuality = c.Query("vq") // Limit search to some accepted tags // Get the tags from the url p.Tags.Parse(c.Query("tags")) // Order to return the results // Getting the order from the "order" argument in url, we default to descending order p.Order = parseOrder(c) // Limit to some status the results // helper to parse status from the "s" argument in url p.Status.Parse(c.Query("s")) // Sort the results // Parse the sorting mode of the result from the "sort" argument in url p.Sort.Parse(c.Query("sort")) // Set NoNull to improve pg query if p.Sort == Date { p.NotNull = p.Sort.ToDBField() + " IS NOT NULL" } // Category in which you have to search // Parse the categories from the "c" argument in url p.Category = ParseCategories(c.Query("c")) // Languages filter of the torrents // We get the languages filtering the results from the "lang" argument in url p.Languages = ParseLanguages(c.QueryArray("lang")) // From which date you need to search and To which date you need to search // maxage is an int parameter limiting the results to the last "x" days (old nyaa behavior) p.FromDate, p.ToDate = backwardCompatibility(c.Query("maxage"), c.Query("fromDate"), c.Query("toDate"), c.Query("dateType")) // Parsing minimum and maximum size from the sizeType given (minSize & maxSize & sizeType in url) // Minimum size to search p.MinSize.Parse(c.Query("minSize"), c.Query("sizeType")) // Maximum size to search p.MaxSize.Parse(c.Query("maxSize"), c.Query("sizeType")) // Needed to display result after a certain torrentID or to limit results to some torrent IDs p.FromID, p.TorrentID = parseTorrentID(c) } // toESQuery : Builds a query string with for es query string query defined here // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html func (p *TorrentParam) toESQuery(c *gin.Context) *Query { query := &Query{ TorrentParam: p, } if len(p.Category) > 0 { query.Append(p.Category.ToESQuery()) } if c.Query("userID") != "" { if !strings.Contains(c.Query("userID"), ",") { if p.UserID > 0 { query.Append("uploader_id:" + strconv.FormatInt(int64(p.UserID), 10)) if p.Hidden { query.Append("hidden:false") } } else if p.UserID == 0 { query.Append(fmt.Sprintf("(uploader_id: %d OR hidden:%t)", p.UserID, true)) } } else { userIDs := strings.Split(c.Query("userID"), ",") for _, str := range userIDs { if userID, err := strconv.Atoi(str); err == nil && userID >= 0 { if userID > 0 { query.Append(fmt.Sprintf("(uploader_id:%d AND hidden:false)", userID)) } else { query.Append(fmt.Sprintf("(uploader_id:%d OR hidden:%t)", userID, true)) } } } } } if c.Query("nuserID") != "" { nuserID := strings.Split(c.Query("nuserID"), ",") for _, str := range nuserID { if userID, err := strconv.Atoi(str); err == nil && userID >= 0 { if userID > 0 { query.Append(fmt.Sprintf("NOT(uploader_id:%d AND hidden:false)", userID)) } else { query.Append(fmt.Sprintf("NOT(uploader_id:%d AND hidden:false) !hidden:true", userID)) } } } } if p.Status != ShowAll { query.Append(p.Status.ToESQuery()) } if p.FromID != 0 { query.Append("id:>" + strconv.FormatInt(int64(p.FromID), 10)) } if len(p.TorrentID) > 0 { for _, id := range p.TorrentID { query.Append(fmt.Sprintf("id:%d", id)) } } if p.FromDate != "" || p.ToDate != "" { query.Append("date: [" + p.FromDate.ToESQuery() + " " + p.ToDate.ToESQuery() + "]") } if p.MinSize > 0 || p.MaxSize > 0 { query.Append("filesize: [" + p.MinSize.ToESQuery() + " " + p.MaxSize.ToESQuery() + "]") } if len(p.Languages) > 0 { langsToESQuery(query, p.Languages) } // Tags search // Anidb if p.AnidbID != 0 { query.Append("anidbid:" + strconv.FormatInt(int64(p.FromID), 10)) } // Vndb if p.VndbID != 0 { query.Append("vndbid:" + strconv.FormatInt(int64(p.FromID), 10)) } // Vgmdb if p.VgmdbID != 0 { query.Append("vgmdbid:" + strconv.FormatInt(int64(p.FromID), 10)) } // Dlsite if p.Dlsite != 0 { query.Append("dlsite:" + strconv.FormatInt(int64(p.FromID), 10)) } // Video quality if p.VideoQuality != "" { query.Append("videoquality:" + p.VideoQuality) } // Other tags if len(p.Tags) > 0 { query.Append(p.Tags.ToESQuery()) } return query } // FindES : /* Uses elasticsearch to find the torrents based on TorrentParam */ func (p *TorrentParam) FindES(c *gin.Context, client *elastic.Client) ([]models.Torrent, int64, error) { search, err := p.toESQuery(c).ToESQuery(client) if err != nil { return nil, 0, err } result, err := search.Do(c) if err != nil { return nil, 0, err } log.Infof("Query '%s' took %d milliseconds.", p.NameLike, result.TookInMillis) log.Infof("Amount of results %d.", result.TotalHits()) var torrents []models.Torrent var torrentCount int if len(result.Hits.Hits) <= 0 { return nil, 0, nil } for _, hit := range result.Hits.Hits { var tJSON models.TorrentJSON err := json.Unmarshal(*hit.Source, &tJSON) if err == nil { torrents = append(torrents, tJSON.ToTorrent()) torrentCount++ } else { log.Errorf("Cannot unmarshal elasticsearch torrent: %s", err) } } if torrentCount < len(result.Hits.Hits) { log.Errorf("Only %d / %d parsed correctly, see error above", torrentCount, len(result.Hits.Hits)) } return torrents, result.TotalHits(), nil } func (p *TorrentParam) toDBQuery(c *gin.Context) *Query { query := &Query{} query.Append(p.Category.ToDBQuery()) if len(p.Languages) > 0 { query.Append("language "+searchOperator, "%"+langsToDBQuery(p.Languages)+"%") } if c.Query("userID") != "" { if !strings.Contains(c.Query("userID"), ",") { if p.UserID > 0 { query.Append("uploader", p.UserID) if p.Hidden { query.Append("hidden", false) } } else if p.UserID == 0 { query.Append("(uploader = ? OR hidden = ?)", p.UserID, true) } } else { userIDs := strings.Split(c.Query("userID"), ",") for _, str := range userIDs { if userID, err := strconv.Atoi(str); err == nil && userID >= 0 { if userID > 0 { query.Append("(uploader = ? AND hidden = ?)", userID, false) } else { query.Append("(uploader = ? OR hidden = ?)", userID, true) } } } } } if c.Query("nuserID") != "" { nuserID := strings.Split(c.Query("nuserID"), ",") for _, str := range nuserID { if userID, err := strconv.Atoi(str); err == nil && userID >= 0 { if userID > 0 { query.Append("NOT(uploader = ? AND hidden = ?)", userID, false) } else { query.Append("uploader <> ? AND hidden != ?", userID, true) } } } } if p.FromID != 0 { query.Append("torrent_id > ?", p.FromID) } if len(p.TorrentID) > 0 { for _, id := range p.TorrentID { query.Append("torrent_id = ?", id) } } if p.FromDate != "" { query.Append("date >= ?", p.FromDate.ToDBQuery()) } if p.ToDate != "" { query.Append("date <= ?", p.ToDate.ToDBQuery()) } if p.Status != 0 { query.Append(p.Status.ToDBQuery(), strconv.Itoa(int(p.Status)+1)) } if len(p.NotNull) > 0 { query.Append(p.NotNull) } if p.MinSize > 0 { query.Append("filesize >= ?", p.MinSize.ToDBQuery()) } if p.MaxSize > 0 { query.Append("filesize <= ?", p.MaxSize.ToDBQuery()) } // Tags search // Anidb if p.AnidbID > 0 { query.Append("anidbid = ?", p.AnidbID) } // Vndb if p.VndbID > 0 { query.Append("vndbid = ?", p.VndbID) } // Vgmdb if p.VgmdbID > 0 { query.Append("vgmdbid = ?", p.VgmdbID) } // Dlsite if p.Dlsite > 0 { query.Append("dlsite = ?", p.Dlsite) } // Video quality if p.VideoQuality != "" { query.Append("videoquality = ?", p.VideoQuality) } // Other tags if len(p.Tags) > 0 { query.Append(p.Tags.ToDBQuery()) } querySplit := strings.Fields(p.NameLike) for _, word := range querySplit { firstRune, _ := utf8.DecodeRuneInString(word) if len(word) == 1 && unicode.IsPunct(firstRune) { // some queries have a single punctuation character // which causes a full scan instead of using the index // and yields no meaningful results. // due to len() == 1 we're just looking at 1-byte/ascii // punctuation characters. continue } if useTSQuery && stringIsASCII(word) { query.Append("torrent_name @@ plainto_tsquery(?)", word) } else { // TODO: possible to make this faster? query.Append("torrent_name "+searchOperator, "%"+word+"%") } } return query } // FindDB : /* Uses SQL to find the torrents based on TorrentParam */ func (p *TorrentParam) FindDB(c *gin.Context) ([]models.Torrent, int64, error) { orderBy := p.Sort.ToDBField() query := p.toDBQuery(c) orderBy += " " switch p.Order { case true: orderBy += "asc" if models.ORM.Dialect().GetName() == "postgres" { orderBy += " NULLS FIRST" } case false: orderBy += "desc" if models.ORM.Dialect().GetName() == "postgres" { orderBy += " NULLS LAST" } } log.Infof("SQL query is :: %s\n", query.String()) if p.Deleted { tor, count, err := torrents.FindDeleted(query, orderBy, int(p.Max), int(uint32(p.Max)*(p.Offset-1))) return tor, int64(count), err } else if p.Full { tor, count, err := torrents.FindWithUserOrderBy(query, orderBy, int(p.Max), int(uint32(p.Max)*(p.Offset-1))) return tor, int64(count), err } tor, count, err := torrents.FindOrderBy(query, orderBy, int(p.Max), int(uint32(p.Max)*(p.Offset-1))) return tor, int64(count), err } // Clone : To clone a torrent params func (p *TorrentParam) Clone() TorrentParam { return TorrentParam{ Order: p.Order, Status: p.Status, Sort: p.Sort, Category: p.Category, Max: p.Max, Offset: p.Offset, UserID: p.UserID, TorrentID: p.TorrentID, FromID: p.FromID, FromDate: p.FromDate, ToDate: p.ToDate, NotNull: p.NotNull, NameLike: p.NameLike, Languages: p.Languages, MinSize: p.MinSize, MaxSize: p.MaxSize, } }