Albirew/nyaa-pantsu
Archivé
1
0
Bifurcation 0
Ce dépôt a été archivé le 2022-05-07. Vous pouvez voir ses fichiers ou le cloner, mais pas ouvrir de ticket ou de demandes d'ajout, ni soumettre de changements.
nyaa-pantsu/utils/validator/torrent/functions.go
akuma06 c6168be8b1 Tag Search + Tests + Search slight refactor [DONE] (#1342)
* Tag Search + Tests + Search slight refactor

First commit improving search. Different struct have their own file with their tests. This way of separating struct by files is inspired by the go packages I've seen so far.

Added new behaviour as discussed in #1334

* fix fallback to ES

* Added some comments to explain PG fallback + log err moved

* Refactored search

Nearly fully covered
WhereParams struct has disappeared for Query struct instead
In DB model, we use an interface implementing Query struct methods

* 1rst Refactor of Tags (WTF already?!)

Prepare Tags for the refactored system. Now there will be descriptive tags for a particular release (ecchi, BDSM, ....) and typed tags.
Typed tags are tags relevant to all torrents and can be limited to some input value. For example, video quality is a typed tag limited to some values (hd, full hd, sd, ...). In the same way, anidbid is also a typed tag but doesn't have default values.

Furthermore, the location storage of tags have changed, now accepted descriptive tags are stored in the torrents table in the column "tags" and they are separated by commas.
In the opposite, accepted typed tags can have have their own column in the torrents table. For example, anidbid, vndbid will populate the column DbID when accepted. On the other hand, videoquality will populate the same way as descriptive tags.

This behaviour depends on the callbackOnType function in tag/helpers.go

* fix for modtools :')

* Added anidb, vndb, dlsite & vmdb id fields in torrent model.
Tags don't have an accepted field anymore.
Accepted Tags are in torrent.AcceptedTags and non-accepted ones in torrrent.Tags.

New Helper + New Changelog for translation string.

* New upload/edit form for torrent tags.
Now the inputs are dynamically generated by the helper tag_form.
No more modal window in those form, only inputs.
Support of tags in API
New translation string for the  link to the modal on torrent view.
More comments in the functions for tags

* Improving how config for tags work. Adding a test on them with understandable messages.
Config for tags have now a Field attribute which is linked to the Torrent model. For example anidbid tag type has now a AnidbID field in config which is the name of the field in torrent model (AnidbID). Every new tag type need to have a field attribute with its counterpart in torrent Model.
Fixing some errors

* Fix compile error + Tests Errors

* Improve performance by caching the list of tags with an index
Adding/removing tags works/tested
New translation strings

TODO: test/fix adding tag on upload/edit

* Mini fix to display video quality
+ tags works/tested on modo edit

* Fix editing tags on modpanel

* Edit tags works

* Add translation string

* Add search backend for tags.
?tags=xxx,eee,ddd
?anidb=21
?vndb=23
?vgmdb=24
?vq=full_hd

* Fix Ajax tag Removal&Add

* Added form for descriptive tags

* Forgot to add the link between database and form for descriptive tags.

* Adding the increase/decrease pantsu for descriptive tags

* Fix #1370

* When you actually forgot to commit files after having forgotten commits
2017-08-22 11:48:10 +10:00

354 lignes
9,4 Kio
Go

package torrentValidator
import (
"encoding/base32"
"encoding/hex"
"fmt"
"io"
"mime/multipart"
"net/url"
"reflect"
"regexp"
"strconv"
"strings"
"github.com/NyaaPantsu/nyaa/config"
"github.com/NyaaPantsu/nyaa/utils/categories"
"github.com/NyaaPantsu/nyaa/utils/cookies"
"github.com/NyaaPantsu/nyaa/utils/format"
msg "github.com/NyaaPantsu/nyaa/utils/messages"
"github.com/NyaaPantsu/nyaa/utils/metainfo"
"github.com/NyaaPantsu/nyaa/utils/torrentLanguages"
"github.com/NyaaPantsu/nyaa/utils/validator/tag"
"github.com/gin-gonic/gin"
"github.com/zeebo/bencode"
)
// ValidateName is a function validating the torrent name
func (r *TorrentRequest) ValidateName() error {
// then actually check that we have everything we need
if len(r.Name) == 0 {
return errTorrentNameInvalid
}
return nil
}
// ValidateTags is a function validating tags by removing duplicate entries
func (r *TorrentRequest) ValidateTags() {
// and filter out multiple tags with the same type (only keep the first one)
var index config.ArrayString
filteredTags := []tagsValidator.CreateForm{}
for _, tag := range r.Tags {
if index.Contains(tag.Type) || tag.Type == "" {
continue
}
filteredTags = append(filteredTags, tag)
index = append(index, tag.Type)
}
r.Tags = filteredTags
}
// ValidateDescription is a function validating description length
func (r *TorrentRequest) ValidateDescription() error {
if len(r.Description) > config.Get().DescriptionLength {
return errTorrentDescInvalid
}
return nil
}
// ValidateMagnet is a function validating a magnet uri format
func (r *TorrentRequest) ValidateMagnet() error {
magnetURL, err := url.Parse(string(r.Magnet)) //?
if err != nil {
return err
}
xt := magnetURL.Query().Get("xt")
if !strings.HasPrefix(xt, "urn:btih:") {
return errTorrentMagnetInvalid
}
xt = strings.SplitAfter(xt, ":")[2]
r.Infohash = strings.TrimSpace(strings.ToUpper(strings.Split(xt, "&")[0]))
return nil
}
// ValidateWebsiteLink is a function validating a website link
func (r *TorrentRequest) ValidateWebsiteLink() error {
if r.WebsiteLink != "" {
// WebsiteLink
urlRegexp, _ := regexp.Compile(`^(https?:\/\/|ircs?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*(\/.*)?$`)
if !urlRegexp.MatchString(r.WebsiteLink) {
return errTorrentURIInvalid
}
}
return nil
}
// ValidateHash is a function validating a torrent hash
func (r *TorrentRequest) ValidateHash() error {
isBase32, err := regexp.MatchString("^[2-7A-Z]{32}$", r.Infohash)
if err != nil {
return err
}
if !isBase32 {
isBase16, err := regexp.MatchString("^[0-9A-F]{40}$", r.Infohash)
if err != nil {
return err
}
if !isBase16 {
return errTorrentHashInvalid
}
} else {
//convert to base16
data, err := base32.StdEncoding.DecodeString(r.Infohash)
if err != nil {
return err
}
hash16 := make([]byte, hex.EncodedLen(len(data)))
hex.Encode(hash16, data)
r.Infohash = strings.ToUpper(string(hash16))
}
return nil
}
// ExtractCategory : takes an http request and computes category field for this form
func (r *TorrentRequest) ExtractCategory() error {
catsSplit := strings.Split(r.Category, "_")
// need this to prevent out of index panics
if len(catsSplit) != 2 {
return errTorrentCatInvalid
}
CatID, err := strconv.Atoi(catsSplit[0])
if err != nil {
return errTorrentCatInvalid
}
SubCatID, err := strconv.Atoi(catsSplit[1])
if err != nil {
return errTorrentCatInvalid
}
if !categories.Exists(r.Category) {
return errTorrentCatInvalid
}
r.CategoryID = CatID
r.SubCategoryID = SubCatID
return nil
}
// ExtractLanguage : takes a http request, computes the torrent language from the form.
func (r *TorrentRequest) ExtractLanguage() error {
isEnglishCategory := false
for _, cat := range config.Get().Torrents.EnglishOnlyCategories {
if cat == r.Category {
isEnglishCategory = true
break
}
}
if len(r.Languages) == 0 {
// If no language, but in an English category, set to en-us, else just stop the check.
if !isEnglishCategory {
return nil
}
r.Languages = append(r.Languages, "en")
return nil
}
englishSelected := false
for _, language := range r.Languages {
if language == "en" {
englishSelected = true
}
if language != "" && !torrentLanguages.LanguageExists(language) {
return errTorrentLangInvalid
}
if strings.HasPrefix(language, "en") && isEnglishCategory {
englishSelected = true
}
}
// We shouldn't return an error for languages, just adding the right language is enough
if !englishSelected && isEnglishCategory {
r.Languages = append(r.Languages, "en")
return nil
}
// We shouldn't return an error if someone has selected only english for languages and missed the right category. Just move the torrent in the right one
// Multiple if conditions so we only do this for loop when needed
if len(r.Languages) == 1 && strings.HasPrefix(r.Languages[0], "en") && !isEnglishCategory && r.CategoryID > 0 {
for key, cat := range config.Get().Torrents.NonEnglishOnlyCategories {
if cat == r.Category {
r.Category = config.Get().Torrents.EnglishOnlyCategories[key]
isEnglishCategory = true
break
}
}
}
return nil
}
// ValidateMultipartUpload : Check if multipart upload is valid
func (r *TorrentRequest) ValidateMultipartUpload(c *gin.Context, uploadFormTorrent string) (multipart.File, error) {
// first: parse torrent file (if any) to fill missing information
tfile, _, err := c.Request.FormFile(uploadFormTorrent)
if err == nil {
var torrent metainfo.TorrentFile
// decode torrent
_, seekErr := tfile.Seek(0, io.SeekStart)
if seekErr != nil {
return tfile, seekErr
}
err = bencode.NewDecoder(tfile).Decode(&torrent)
if err != nil {
return tfile, metainfo.ErrInvalidTorrentFile
}
// check a few things
if torrent.IsPrivate() {
return tfile, errTorrentPrivate
}
trackers := torrent.GetAllAnnounceURLS()
r.Trackers = CheckTrackers(trackers)
if len(r.Trackers) == 0 {
return tfile, errTorrentNoTrackers
}
// Name
if len(r.Name) == 0 {
r.Name = torrent.TorrentName()
}
// Magnet link: if a file is provided it should be empty
if len(r.Magnet) != 0 {
return tfile, errTorrentAndMagnet
}
_, seekErr = tfile.Seek(0, io.SeekStart)
if seekErr != nil {
return tfile, seekErr
}
infohash, err := metainfo.DecodeInfohash(tfile)
if err != nil {
return tfile, metainfo.ErrInvalidTorrentFile
}
r.Infohash = infohash
r.Magnet = format.InfoHashToMagnet(infohash, r.Name, trackers...)
// extract filesize
r.Filesize = int64(torrent.TotalSize())
// extract filelist
fileInfos := torrent.Info.GetFiles()
for _, fileInfo := range fileInfos {
r.FileList = append(r.FileList, uploadedFile{
Path: fileInfo.Path,
Filesize: int64(fileInfo.Length),
})
}
} else {
err = r.ValidateMagnet()
if err != nil {
return tfile, err
}
err = r.ValidateHash()
if err != nil {
return tfile, err
}
// TODO: Get Trackers from magnet URL
r.Filesize = 0
r.Filepath = ""
return tfile, nil
}
return tfile, err
}
// ExtractInfo : Function to assign values from request to ReassignForm
func (f *ReassignForm) ExtractInfo(c *gin.Context) bool {
f.By = c.PostForm("by")
messages := msg.GetMessages(c)
if f.By != "olduser" && f.By != "torrentid" {
messages.AddErrorTf("errors", "no_action_exist", f.By)
return false
}
f.Data = strings.Trim(c.PostForm("data"), " \r\n")
if f.By == "olduser" {
if f.Data == "" {
messages.AddErrorT("errors", "user_not_found")
return false
} else if strings.Contains(f.Data, "\n") {
messages.AddErrorT("errors", "multiple_username_error")
return false
}
} else if f.By == "torrentid" {
if f.Data == "" {
messages.AddErrorT("errors", "no_id_given")
return false
}
splitData := strings.Split(f.Data, "\n")
for i, tmp := range splitData {
tmp = strings.Trim(tmp, " \r")
torrentID, err := strconv.ParseUint(tmp, 10, 0)
if err != nil {
messages.AddErrorTf("errors", "parse_error_line", i+1)
return false // TODO: Shouldn't it continue to parse the rest and display the errored lines?
}
f.Torrents = append(f.Torrents, uint(torrentID))
}
}
tmpID := c.PostForm("to")
parsed, err := strconv.ParseUint(tmpID, 10, 32)
if err != nil {
messages.Error(err)
return false
}
f.AssignTo = uint(parsed)
_, _, _, _, err = cookies.RetrieveUserFromRequest(c, uint(parsed))
if err != nil {
messages.AddErrorTf("errors", "no_user_found_id", int(parsed))
return false
}
return true
}
// Get check if the tag map has the same tag in it (tag value + tag type)
func (ts TagsRequest) Get(tagType string) tagsValidator.CreateForm {
for _, ta := range ts {
if ta.Type == tagType {
return ta
}
}
return tagsValidator.CreateForm{}
}
type torrentInt interface {
GetDescriptiveTags() string
}
// Bind a torrent model with its tags to the tags request
func (ts *TagsRequest) Bind(torrent torrentInt) error {
for _, tagConf := range config.Get().Torrents.Tags.Types {
if tagConf.Field == "" {
return errMissingFieldConfig
}
tagField := reflect.ValueOf(torrent).Elem().FieldByName(tagConf.Field)
if !tagField.IsValid() {
return errWrongFieldConfig
}
if fmt.Sprint(tagField.Interface()) != "" && fmt.Sprint(tagField.Interface()) != "0" {
*ts = append(*ts, tagsValidator.CreateForm{Type: tagConf.Name, Tag: fmt.Sprint(tagField.Interface())})
}
}
if torrent.GetDescriptiveTags() != "" {
*ts = append(*ts, tagsValidator.CreateForm{Type: config.Get().Torrents.Tags.Default, Tag: torrent.GetDescriptiveTags()})
}
return nil
}