Albirew/nyaa-pantsu
Archivé
1
0
Bifurcation 0
Ce dépôt a été archivé le 2022-05-07. Vous pouvez voir ses fichiers ou le cloner, mais pas ouvrir de ticket ou de demandes d'ajout, ni soumettre de changements.
nyaa-pantsu/common/torrent.go
Ramon Dantas d8e17478f8 Country flags (language) for torrents. (#970)
* Add flags for torrents

Add a new field, .Language, to the Torrent model, and a new package,
torrentLanguages, which maps languages to flags. Added also a flag icon pack
from googlei18n/region-flags, with (mostly) public domain flags from Wikipedia.

* Optimize flags

* Use FlagSprites CSS instead of .png files

* Only use flags for languages we support

* Add test for CSS flags

Ensure that we have all the flags for the languages we support.

* Add AdditionalLanguages field to config

This allows us to support additional languages for new uploaded torrents,
even if we have no translation for it.

* Minor CSS fix

* Add "other" and "multiple" torrent languages

Also removed the TorrentLanguage struct, as it wasn't much useful.

* Fix test

* Add colspan=2 to category when language is empty

Also hide the language column if empty.

* Add lang field to search.

Hopefully it works with Elasticsearch as well, but I haven't tested
(lol Java)

* Add language field to ES index and settings

* Add language column to JS template

* Add keyword type to language ES field

* Remove 'raw' from keyword

* Set "simple" analyzer on language

* Document .Language field on Torrent model
2017-06-12 09:14:26 +10:00

243 lignes
6,2 Kio
Go

package common
import (
"context"
"encoding/json"
"net/http"
"strconv"
"strings"
"time"
"github.com/gorilla/mux"
elastic "gopkg.in/olivere/elastic.v5"
"github.com/NyaaPantsu/nyaa/config"
"github.com/NyaaPantsu/nyaa/model"
"github.com/NyaaPantsu/nyaa/util/log"
)
// TorrentParam defines all parameters that can be provided when searching for a torrent
type TorrentParam struct {
All bool // True means ignore everything but Max and Offset
Full bool // True means load all members
Order bool // True means ascending
Status Status
Sort SortMode
Category Category
Max uint32
Offset uint32
UserID uint32
TorrentID uint32
FromID uint32
FromDate string
ToDate string
NotNull string // csv
Null string // csv
NameLike string // csv
Language string
}
// FromRequest : parse a request in torrent param
// TODO Should probably return an error ?
func (p *TorrentParam) FromRequest(r *http.Request) {
var err error
nameLike := strings.TrimSpace(r.URL.Query().Get("q"))
page := mux.Vars(r)["page"]
pagenum, err := strconv.ParseUint(page, 10, 32)
if err != nil {
pagenum = 1
}
max, err := strconv.ParseUint(r.URL.Query().Get("max"), 10, 32)
if err != nil {
max = uint64(config.Conf.Navigation.TorrentsPerPage)
} else if max > uint64(config.Conf.Navigation.MaxTorrentsPerPage) {
max = uint64(config.Conf.Navigation.MaxTorrentsPerPage)
}
// FIXME 0 means no userId defined
userID, err := strconv.ParseUint(r.URL.Query().Get("userID"), 10, 32)
if err != nil {
userID = 0
}
// FIXME 0 means no userId defined
fromID, err := strconv.ParseUint(r.URL.Query().Get("fromID"), 10, 32)
if err != nil {
fromID = 0
}
var status Status
status.Parse(r.URL.Query().Get("s"))
maxage, err := strconv.Atoi(r.URL.Query().Get("maxage"))
if err != nil {
p.FromDate = r.URL.Query().Get("fromDate")
p.ToDate = r.URL.Query().Get("toDate")
} else {
p.FromDate = time.Now().AddDate(0, 0, -maxage).Format("2006-01-02")
}
var category Category
category.Parse(r.URL.Query().Get("c"))
var sortMode SortMode
sortMode.Parse(r.URL.Query().Get("sort"))
ascending := false
if r.URL.Query().Get("order") == "true" {
ascending = true
}
language := strings.TrimSpace(r.URL.Query().Get("lang"))
p.NameLike = nameLike
p.Offset = uint32(pagenum)
p.Max = uint32(max)
p.UserID = uint32(userID)
// TODO Use All
p.All = false
// TODO Use Full
p.Full = false
p.Order = ascending
p.Status = status
p.Sort = sortMode
p.Category = category
p.Language = language
// FIXME 0 means no TorrentId defined
// Do we even need that ?
p.TorrentID = 0
// Needed to display result after a certain torrentID
p.FromID = uint32(fromID)
}
// ToFilterQuery : Builds a query string with for es query string query defined here
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html
func (p *TorrentParam) ToFilterQuery() string {
// Don't set sub category unless main category is set
query := ""
if p.Category.IsMainSet() {
query += "category:" + strconv.FormatInt(int64(p.Category.Main), 10)
if p.Category.IsSubSet() {
query += " sub_category:" + strconv.FormatInt(int64(p.Category.Sub), 10)
}
}
if p.UserID != 0 {
query += " uploader_id:" + strconv.FormatInt(int64(p.UserID), 10)
}
if p.Status != ShowAll {
if p.Status != FilterRemakes {
query += " status:" + p.Status.ToString()
} else {
/* From the old nyaa behavior, FilterRemake means everything BUT
* remakes
*/
query += " !status:" + p.Status.ToString()
}
}
if p.FromID != 0 {
query += " id:>" + strconv.FormatInt(int64(p.FromID), 10)
}
if p.FromDate != "" && p.ToDate != "" {
query += " date: [" + p.FromDate + " " + p.ToDate + "]"
} else if p.FromDate != "" {
query += " date: [" + p.FromDate + " *]"
} else if p.ToDate != "" {
query += " date: [* " + p.ToDate + "]"
}
if p.Language != "" {
query += " language:" + p.Language
}
return query
}
// Find :
/* Uses elasticsearch to find the torrents based on TorrentParam
* We decided to fetch only the ids from ES and then query these ids to the
* database
*/
func (p *TorrentParam) Find(client *elastic.Client) (int64, []model.Torrent, error) {
// TODO Why is it needed, what does it do ?
ctx := context.Background()
var query elastic.Query
if p.NameLike == "" {
query = elastic.NewMatchAllQuery()
} else {
query = elastic.NewSimpleQueryStringQuery(p.NameLike).
Field("name").
Analyzer(config.Conf.Search.ElasticsearchAnalyzer).
DefaultOperator("AND")
}
// TODO Find a better way to keep in sync with mapping in ansible
search := client.Search().
Index(config.Conf.Search.ElasticsearchIndex).
Query(query).
Type(config.Conf.Search.ElasticsearchType).
From(int((p.Offset-1)*p.Max)).
Size(int(p.Max)).
Sort(p.Sort.ToESField(), p.Order).
Sort("_score", false) // Don't put _score before the field sort, it messes with the sorting
filterQueryString := p.ToFilterQuery()
if filterQueryString != "" {
filterQuery := elastic.NewQueryStringQuery(filterQueryString).
DefaultOperator("AND")
search = search.PostFilter(filterQuery)
}
result, err := search.Do(ctx)
if err != nil {
return 0, nil, err
}
log.Infof("Query '%s' took %d milliseconds.", p.NameLike, result.TookInMillis)
log.Infof("Amount of results %d.", result.TotalHits())
torrents := make([]model.Torrent, len(result.Hits.Hits))
if len(result.Hits.Hits) <= 0 {
return 0, nil, nil
}
for i, hit := range result.Hits.Hits {
// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
var tJson model.TorrentJSON
err := json.Unmarshal(*hit.Source, &tJson)
if err != nil {
log.Errorf("Cannot unmarshal elasticsearch torrent: %s", err)
}
torrent := tJson.ToTorrent()
torrents[i] = torrent
}
return result.TotalHits(), torrents, nil
}
// Clone : To clone a torrent params
func (p *TorrentParam) Clone() TorrentParam {
return TorrentParam{
Order: p.Order,
Status: p.Status,
Sort: p.Sort,
Category: p.Category,
Max: p.Max,
Offset: p.Offset,
UserID: p.UserID,
TorrentID: p.TorrentID,
FromID: p.FromID,
FromDate: p.FromDate,
ToDate: p.ToDate,
NotNull: p.NotNull,
Null: p.Null,
NameLike: p.NameLike,
Language: p.Language,
}
}