c6168be8b1
* Tag Search + Tests + Search slight refactor First commit improving search. Different struct have their own file with their tests. This way of separating struct by files is inspired by the go packages I've seen so far. Added new behaviour as discussed in #1334 * fix fallback to ES * Added some comments to explain PG fallback + log err moved * Refactored search Nearly fully covered WhereParams struct has disappeared for Query struct instead In DB model, we use an interface implementing Query struct methods * 1rst Refactor of Tags (WTF already?!) Prepare Tags for the refactored system. Now there will be descriptive tags for a particular release (ecchi, BDSM, ....) and typed tags. Typed tags are tags relevant to all torrents and can be limited to some input value. For example, video quality is a typed tag limited to some values (hd, full hd, sd, ...). In the same way, anidbid is also a typed tag but doesn't have default values. Furthermore, the location storage of tags have changed, now accepted descriptive tags are stored in the torrents table in the column "tags" and they are separated by commas. In the opposite, accepted typed tags can have have their own column in the torrents table. For example, anidbid, vndbid will populate the column DbID when accepted. On the other hand, videoquality will populate the same way as descriptive tags. This behaviour depends on the callbackOnType function in tag/helpers.go * fix for modtools :') * Added anidb, vndb, dlsite & vmdb id fields in torrent model. Tags don't have an accepted field anymore. Accepted Tags are in torrent.AcceptedTags and non-accepted ones in torrrent.Tags. New Helper + New Changelog for translation string. * New upload/edit form for torrent tags. Now the inputs are dynamically generated by the helper tag_form. No more modal window in those form, only inputs. Support of tags in API New translation string for the link to the modal on torrent view. More comments in the functions for tags * Improving how config for tags work. Adding a test on them with understandable messages. Config for tags have now a Field attribute which is linked to the Torrent model. For example anidbid tag type has now a AnidbID field in config which is the name of the field in torrent model (AnidbID). Every new tag type need to have a field attribute with its counterpart in torrent Model. Fixing some errors * Fix compile error + Tests Errors * Improve performance by caching the list of tags with an index Adding/removing tags works/tested New translation strings TODO: test/fix adding tag on upload/edit * Mini fix to display video quality + tags works/tested on modo edit * Fix editing tags on modpanel * Edit tags works * Add translation string * Add search backend for tags. ?tags=xxx,eee,ddd ?anidb=21 ?vndb=23 ?vgmdb=24 ?vq=full_hd * Fix Ajax tag Removal&Add * Added form for descriptive tags * Forgot to add the link between database and form for descriptive tags. * Adding the increase/decrease pantsu for descriptive tags * Fix #1370 * When you actually forgot to commit files after having forgotten commits
439 lignes
12 Kio
Go
439 lignes
12 Kio
Go
package search
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
elastic "gopkg.in/olivere/elastic.v5"
|
|
|
|
"github.com/NyaaPantsu/nyaa/models"
|
|
"github.com/NyaaPantsu/nyaa/models/torrents"
|
|
"github.com/NyaaPantsu/nyaa/utils/log"
|
|
"github.com/NyaaPantsu/nyaa/utils/publicSettings"
|
|
"github.com/gin-gonic/gin"
|
|
)
|
|
|
|
// TorrentParam defines all parameters that can be provided when searching for a torrent
|
|
type TorrentParam struct {
|
|
Full bool // True means load all members
|
|
Order bool // True means ascending
|
|
Hidden bool // True means filter hidden torrents
|
|
Deleted bool // False means filter deleted torrents
|
|
Status Status
|
|
Sort SortMode
|
|
Category Categories
|
|
Max maxType
|
|
Offset uint32
|
|
UserID uint32
|
|
TorrentID []uint32
|
|
FromID uint32
|
|
FromDate DateFilter
|
|
ToDate DateFilter
|
|
NotNull string // csv
|
|
NameLike string // csv
|
|
Languages publicSettings.Languages
|
|
MinSize SizeBytes
|
|
MaxSize SizeBytes
|
|
// Tags search
|
|
AnidbID uint32
|
|
VndbID uint32
|
|
VgmdbID uint32
|
|
Dlsite uint32
|
|
VideoQuality string
|
|
Tags Tags
|
|
}
|
|
|
|
// Identifier returns a unique identifier for the struct
|
|
func (p *TorrentParam) Identifier() string {
|
|
cats := ""
|
|
for _, v := range p.Category {
|
|
cats += fmt.Sprintf("%d%d", v.Main, v.Sub)
|
|
}
|
|
languages := ""
|
|
for _, v := range p.Languages {
|
|
languages += fmt.Sprintf("%s%s", v.Code, v.Name)
|
|
}
|
|
ids := ""
|
|
for _, v := range p.TorrentID {
|
|
ids += fmt.Sprintf("%d", v)
|
|
}
|
|
// Tags identifier
|
|
tags := strings.Join(p.Tags, ",")
|
|
tags += p.VideoQuality
|
|
dbids := fmt.Sprintf("%d%d%d%d", p.AnidbID, p.VndbID, p.VgmdbID, p.Dlsite)
|
|
|
|
identifier := fmt.Sprintf("%s%s%s%d%d%d%d%d%d%d%s%s%s%d%s%s%s%t%t%t%t", p.NameLike, p.NotNull, languages, p.Max, p.Offset, p.FromID, p.MinSize, p.MaxSize, p.Status, p.Sort, dbids, p.FromDate, p.ToDate, p.UserID, ids, cats, tags, p.Full, p.Order, p.Hidden, p.Deleted)
|
|
return base64.URLEncoding.EncodeToString([]byte(identifier))
|
|
}
|
|
|
|
func parseUInt(c *gin.Context, key string) uint32 {
|
|
// Get the user id from the url
|
|
u64, err := strconv.ParseUint(c.Query(key), 10, 32)
|
|
if err != nil {
|
|
// if you can't convert it, you set it to 0
|
|
u64 = 0
|
|
}
|
|
return uint32(u64)
|
|
}
|
|
func parseTorrentID(c *gin.Context) (uint32, []uint32) {
|
|
// Get the torrent ID to limit the results to the ones after this torrent
|
|
fromID, err := strconv.ParseUint(c.Query("fromID"), 10, 32)
|
|
if err != nil {
|
|
// if you can't convert it, you set it to 0
|
|
fromID = 0
|
|
}
|
|
var torrentIDs []uint32
|
|
ids := c.QueryArray("id")
|
|
|
|
for _, id := range ids {
|
|
idInt, err := strconv.Atoi(id)
|
|
if err == nil {
|
|
torrentIDs = append(torrentIDs, uint32(idInt))
|
|
}
|
|
}
|
|
return uint32(fromID), torrentIDs
|
|
}
|
|
|
|
func parseOrder(c *gin.Context) bool {
|
|
if c.Query("order") == "true" {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// FromRequest : parse a request in torrent param
|
|
// TODO Should probably return an error ?
|
|
func (p *TorrentParam) FromRequest(c *gin.Context) {
|
|
// Search by name
|
|
// We take the search arguments from "q" in url
|
|
p.NameLike = strings.TrimSpace(c.Query("q"))
|
|
|
|
// Maximum results returned
|
|
// We take the maxximum results to display from "limit" in url
|
|
p.Max.Parse(c.Query("limit"))
|
|
|
|
// Limit search to one user
|
|
// Get the user id from the url
|
|
p.UserID = parseUInt(c, "userID")
|
|
|
|
// Limit search to DbID
|
|
// Get the id from the url
|
|
p.AnidbID = parseUInt(c, "anidb")
|
|
p.VndbID = parseUInt(c, "vndb")
|
|
p.VgmdbID = parseUInt(c, "vgm")
|
|
p.Dlsite = parseUInt(c, "dlsite")
|
|
|
|
// Limit search to video quality
|
|
// Get the video quality from url
|
|
p.VideoQuality = c.Query("vq")
|
|
|
|
// Limit search to some accepted tags
|
|
// Get the tags from the url
|
|
p.Tags.Parse(c.Query("tags"))
|
|
|
|
// Order to return the results
|
|
// Getting the order from the "order" argument in url, we default to descending order
|
|
p.Order = parseOrder(c)
|
|
|
|
// Limit to some status the results
|
|
// helper to parse status from the "s" argument in url
|
|
p.Status.Parse(c.Query("s"))
|
|
|
|
// Sort the results
|
|
// Parse the sorting mode of the result from the "sort" argument in url
|
|
p.Sort.Parse(c.Query("sort"))
|
|
|
|
// Category in which you have to search
|
|
// Parse the categories from the "c" argument in url
|
|
p.Category = ParseCategories(c.Query("c"))
|
|
|
|
// Languages filter of the torrents
|
|
// We get the languages filtering the results from the "lang" argument in url
|
|
p.Languages = ParseLanguages(c.QueryArray("lang"))
|
|
|
|
// From which date you need to search and To which date you need to search
|
|
// maxage is an int parameter limiting the results to the last "x" days (old nyaa behavior)
|
|
p.FromDate, p.ToDate = backwardCompatibility(c.Query("maxage"), c.Query("fromDate"), c.Query("toDate"), c.Query("dateType"))
|
|
|
|
// Parsing minimum and maximum size from the sizeType given (minSize & maxSize & sizeType in url)
|
|
// Minimum size to search
|
|
p.MinSize.Parse(c.Query("minSize"), c.Query("sizeType"))
|
|
// Maximum size to search
|
|
p.MaxSize.Parse(c.Query("maxSize"), c.Query("sizeType"))
|
|
|
|
// Needed to display result after a certain torrentID or to limit results to some torrent IDs
|
|
p.FromID, p.TorrentID = parseTorrentID(c)
|
|
}
|
|
|
|
// toESQuery : Builds a query string with for es query string query defined here
|
|
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html
|
|
func (p *TorrentParam) toESQuery(c *gin.Context) *Query {
|
|
query := &Query{
|
|
TorrentParam: p,
|
|
}
|
|
|
|
if len(p.Category) > 0 {
|
|
query.Append(p.Category.ToESQuery())
|
|
}
|
|
|
|
if c.Query("userID") != "" {
|
|
if p.UserID > 0 {
|
|
query.Append("uploader_id:" + strconv.FormatInt(int64(p.UserID), 10))
|
|
if p.Hidden {
|
|
query.Append("hidden:false")
|
|
}
|
|
} else if p.UserID == 0 {
|
|
query.Append(fmt.Sprintf("(uploader_id: %d OR hidden:%t)", p.UserID, true))
|
|
}
|
|
}
|
|
|
|
if p.Status != ShowAll {
|
|
query.Append(p.Status.ToESQuery())
|
|
}
|
|
|
|
if p.FromID != 0 {
|
|
query.Append("id:>" + strconv.FormatInt(int64(p.FromID), 10))
|
|
}
|
|
|
|
if len(p.TorrentID) > 0 {
|
|
for _, id := range p.TorrentID {
|
|
query.Append(fmt.Sprintf("id:%d", id))
|
|
}
|
|
}
|
|
|
|
if p.FromDate != "" || p.ToDate != "" {
|
|
query.Append("date: [" + p.FromDate.ToESQuery() + " " + p.ToDate.ToESQuery() + "]")
|
|
}
|
|
|
|
if p.MinSize > 0 || p.MaxSize > 0 {
|
|
query.Append("filesize: [" + p.MinSize.ToESQuery() + " " + p.MaxSize.ToESQuery() + "]")
|
|
}
|
|
|
|
if len(p.Languages) > 0 {
|
|
langsToESQuery(query, p.Languages)
|
|
}
|
|
|
|
// Tags search
|
|
// Anidb
|
|
if p.AnidbID != 0 {
|
|
query.Append("anidbid:" + strconv.FormatInt(int64(p.FromID), 10))
|
|
}
|
|
// Vndb
|
|
if p.VndbID != 0 {
|
|
query.Append("vndbid:" + strconv.FormatInt(int64(p.FromID), 10))
|
|
}
|
|
// Vgmdb
|
|
if p.VgmdbID != 0 {
|
|
query.Append("vgmdbid:" + strconv.FormatInt(int64(p.FromID), 10))
|
|
}
|
|
// Dlsite
|
|
if p.Dlsite != 0 {
|
|
query.Append("dlsite:" + strconv.FormatInt(int64(p.FromID), 10))
|
|
}
|
|
// Video quality
|
|
if p.VideoQuality != "" {
|
|
query.Append("videoquality:" + p.VideoQuality)
|
|
}
|
|
// Other tags
|
|
if len(p.Tags) > 0 {
|
|
query.Append(p.Tags.ToESQuery())
|
|
}
|
|
|
|
return query
|
|
}
|
|
|
|
// FindES :
|
|
/* Uses elasticsearch to find the torrents based on TorrentParam
|
|
*/
|
|
func (p *TorrentParam) FindES(c *gin.Context, client *elastic.Client) ([]models.Torrent, int64, error) {
|
|
search, err := p.toESQuery(c).ToESQuery(client)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
result, err := search.Do(c)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
log.Infof("Query '%s' took %d milliseconds.", p.NameLike, result.TookInMillis)
|
|
log.Infof("Amount of results %d.", result.TotalHits())
|
|
|
|
var torrents []models.Torrent
|
|
var torrentCount int
|
|
if len(result.Hits.Hits) <= 0 {
|
|
return nil, 0, nil
|
|
}
|
|
for _, hit := range result.Hits.Hits {
|
|
var tJSON models.TorrentJSON
|
|
err := json.Unmarshal(*hit.Source, &tJSON)
|
|
if err == nil {
|
|
torrents = append(torrents, tJSON.ToTorrent())
|
|
torrentCount++
|
|
} else {
|
|
log.Errorf("Cannot unmarshal elasticsearch torrent: %s", err)
|
|
}
|
|
}
|
|
if torrentCount < len(result.Hits.Hits) {
|
|
log.Errorf("Only %d / %d parsed correctly, see error above", torrentCount, len(result.Hits.Hits))
|
|
}
|
|
|
|
return torrents, result.TotalHits(), nil
|
|
}
|
|
|
|
func (p *TorrentParam) toDBQuery(c *gin.Context) *Query {
|
|
query := &Query{}
|
|
|
|
query.Append(p.Category.ToDBQuery())
|
|
if len(p.Languages) > 0 {
|
|
query.Append("language "+searchOperator, "%"+langsToDBQuery(p.Languages)+"%")
|
|
}
|
|
|
|
if c.Query("userID") != "" {
|
|
if p.UserID > 0 {
|
|
query.Append("uploader", p.UserID)
|
|
if p.Hidden {
|
|
query.Append("hidden", false)
|
|
}
|
|
} else if p.UserID == 0 {
|
|
query.Append("(uploader = ? OR hidden = ?)", p.UserID, true)
|
|
}
|
|
}
|
|
if p.FromID != 0 {
|
|
query.Append("torrent_id > ?", p.FromID)
|
|
}
|
|
if len(p.TorrentID) > 0 {
|
|
for _, id := range p.TorrentID {
|
|
query.Append("torrent_id = ?", id)
|
|
}
|
|
}
|
|
if p.FromDate != "" {
|
|
query.Append("date >= ?", p.FromDate.ToDBQuery())
|
|
}
|
|
if p.ToDate != "" {
|
|
query.Append("date <= ?", p.ToDate.ToDBQuery())
|
|
}
|
|
if p.Status != 0 {
|
|
query.Append(p.Status.ToDBQuery(), strconv.Itoa(int(p.Status)+1))
|
|
}
|
|
if len(p.NotNull) > 0 {
|
|
query.Append(p.NotNull)
|
|
}
|
|
if p.MinSize > 0 {
|
|
query.Append("filesize >= ?", p.MinSize.ToDBQuery())
|
|
}
|
|
if p.MaxSize > 0 {
|
|
query.Append("filesize <= ?", p.MaxSize.ToDBQuery())
|
|
}
|
|
|
|
// Tags search
|
|
// Anidb
|
|
if p.AnidbID > 0 {
|
|
query.Append("anidbid = ?", p.AnidbID)
|
|
}
|
|
// Vndb
|
|
if p.VndbID > 0 {
|
|
query.Append("vndbid = ?", p.VndbID)
|
|
}
|
|
// Vgmdb
|
|
if p.VgmdbID > 0 {
|
|
query.Append("vgmdbid = ?", p.VgmdbID)
|
|
}
|
|
// Dlsite
|
|
if p.Dlsite > 0 {
|
|
query.Append("dlsite = ?", p.Dlsite)
|
|
}
|
|
// Video quality
|
|
if p.VideoQuality != "" {
|
|
query.Append("videoquality = ?", p.VideoQuality)
|
|
}
|
|
// Other tags
|
|
if len(p.Tags) > 0 {
|
|
query.Append(p.Tags.ToDBQuery())
|
|
}
|
|
|
|
querySplit := strings.Fields(p.NameLike)
|
|
for _, word := range querySplit {
|
|
firstRune, _ := utf8.DecodeRuneInString(word)
|
|
if len(word) == 1 && unicode.IsPunct(firstRune) {
|
|
// some queries have a single punctuation character
|
|
// which causes a full scan instead of using the index
|
|
// and yields no meaningful results.
|
|
// due to len() == 1 we're just looking at 1-byte/ascii
|
|
// punctuation characters.
|
|
continue
|
|
}
|
|
|
|
if useTSQuery && stringIsASCII(word) {
|
|
query.Append("torrent_name @@ plainto_tsquery(?)", word)
|
|
} else {
|
|
// TODO: possible to make this faster?
|
|
query.Append("torrent_name "+searchOperator, "%"+word+"%")
|
|
}
|
|
}
|
|
return query
|
|
}
|
|
|
|
// FindDB :
|
|
/* Uses SQL to find the torrents based on TorrentParam
|
|
*/
|
|
func (p *TorrentParam) FindDB(c *gin.Context) ([]models.Torrent, int64, error) {
|
|
orderBy := p.Sort.ToDBField()
|
|
if p.Sort == Date {
|
|
p.NotNull = p.Sort.ToDBField() + " IS NOT NULL"
|
|
}
|
|
query := p.toDBQuery(c)
|
|
orderBy += " "
|
|
|
|
switch p.Order {
|
|
case true:
|
|
orderBy += "asc"
|
|
if models.ORM.Dialect().GetName() == "postgres" {
|
|
orderBy += " NULLS FIRST"
|
|
}
|
|
case false:
|
|
orderBy += "desc"
|
|
if models.ORM.Dialect().GetName() == "postgres" {
|
|
orderBy += " NULLS LAST"
|
|
}
|
|
}
|
|
|
|
log.Infof("SQL query is :: %s\n", query.String())
|
|
|
|
if p.Deleted {
|
|
tor, count, err := torrents.FindDeleted(query, orderBy, int(p.Max), int(uint32(p.Max)*(p.Offset-1)))
|
|
return tor, int64(count), err
|
|
} else if p.Full {
|
|
tor, count, err := torrents.FindWithUserOrderBy(query, orderBy, int(p.Max), int(uint32(p.Max)*(p.Offset-1)))
|
|
return tor, int64(count), err
|
|
}
|
|
tor, count, err := torrents.FindOrderBy(query, orderBy, int(p.Max), int(uint32(p.Max)*(p.Offset-1)))
|
|
return tor, int64(count), err
|
|
}
|
|
|
|
// Clone : To clone a torrent params
|
|
func (p *TorrentParam) Clone() TorrentParam {
|
|
return TorrentParam{
|
|
Order: p.Order,
|
|
Status: p.Status,
|
|
Sort: p.Sort,
|
|
Category: p.Category,
|
|
Max: p.Max,
|
|
Offset: p.Offset,
|
|
UserID: p.UserID,
|
|
TorrentID: p.TorrentID,
|
|
FromID: p.FromID,
|
|
FromDate: p.FromDate,
|
|
ToDate: p.ToDate,
|
|
NotNull: p.NotNull,
|
|
NameLike: p.NameLike,
|
|
Languages: p.Languages,
|
|
MinSize: p.MinSize,
|
|
MaxSize: p.MaxSize,
|
|
}
|
|
}
|