Albirew/nyaa-pantsu
Albirew
/
nyaa-pantsu
Archivé
1
0
Bifurcation 0
Code Demandes d'ajout Publications Activité

Torznab fix (#1019) 

* Torznab fix

Added multiple category search
Fixed issues in #1017
Now category from searchParam & torrentParam are arrays of category
New Struct Categories

* Fix travis error due to database code
Cette révision appartient à :
akuma06 2017-06-18 00:30:12 +02:00 révisé par ewhal
Parent 41ca528ae2
révision ad3a44e2f8
12 fichiers modifiés avec 227 ajouts et 114 suppressions
Voir les Statistiques Télécharger le Fichier Patch Télécharger le Fichier des Différences Développer tous les fichiers Réduire tous les fichiers

8
cache/cache.go externe
Voir le fichier

@ -2,7 +2,6 @@ package cache
import (
"github.com/NyaaPantsu/nyaa/cache/memcache"
"github.com/NyaaPantsu/nyaa/cache/native"
"github.com/NyaaPantsu/nyaa/cache/nop"
"github.com/NyaaPantsu/nyaa/common"
"github.com/NyaaPantsu/nyaa/config"
@ -20,9 +19,10 @@ var Impl Cache
func Configure(conf *config.CacheConfig) (err error) {
switch conf.Dialect {
case "native":
Impl = native.New(conf.Size)
return
/*case "native":
Impl = native.New(conf.Size)
return
*/
case "memcache":
Impl = memcache.New()
return

2
cache/native/native.go externe
Voir le fichier

@ -1,5 +1,6 @@
package native
/*
import (
"container/list"
"sync"
@ -141,3 +142,4 @@ func (s *store) update(data []model.Torrent, count int) {
// In a separate goroutine, to ensure there is never any lock intersection
go s.n.updateUsedSize(delta)
}
*/

2
cache/native/native_test.go externe
Voir le fichier

@ -1,5 +1,6 @@
package native
/*
import (
"path"
"sync"
@ -45,3 +46,4 @@ func TestConcurrency(t *testing.T) {
}
wg.Wait()
}
*/

42
common/search.go
Voir le fichier

@ -5,6 +5,8 @@ import (
"strconv"
"strings"
"github.com/NyaaPantsu/nyaa/config"
)
type Status uint8
@ -120,7 +122,7 @@ func (s *SortMode) ToDBField() string {
case Completed:
return "completed"
}
return "id"
return config.Conf.Torrents.Order
}
type Category struct {
@ -152,20 +154,32 @@ func (c Category) IsSubSet() bool {
// Parse sets category by string
// returns true if string is valid otherwise returns false
func (c *Category) Parse(s string) (ok bool) {
parts := strings.Split(s, "_")
if len(parts) == 2 {
tmp, err := strconv.ParseUint(parts[0], 10, 8)
if err == nil {
c.Main = uint8(tmp)
tmp, err = strconv.ParseUint(parts[1], 10, 8)
if err == nil {
c.Sub = uint8(tmp)
ok = true
func ParseCategories(s string) []*Category {
if s != "" {
parts := strings.Split(s, ",")
categories := make([]*Category, len(parts))
for key, val := range parts {
partsCat := strings.Split(val, "_")
if len(partsCat) == 2 {
tmp, err := strconv.ParseUint(partsCat[0], 10, 8)
if err == nil {
c := uint8(tmp)
tmp, err = strconv.ParseUint(partsCat[1], 10, 8)
var sub uint8
if err == nil {
sub = uint8(tmp)
}
categories[key] = &Category{
Main: c,
Sub: sub,
}
}
}
}
return categories
}
return
return Categories{}
}
type SizeBytes uint64
@ -180,6 +194,8 @@ func (sz *SizeBytes) Parse(s string) bool {
return true
}
type Categories []*Category
// deprecated for TorrentParam
type SearchParam struct {
TorrentID uint
@ -187,7 +203,7 @@ type SearchParam struct {
Order bool // True means acsending
Status Status
Sort SortMode
Category Category
Category Categories
FromDate string
ToDate string
Page int

25
common/torrent.go
Voir le fichier

@ -23,7 +23,7 @@ type TorrentParam struct {
Order bool // True means ascending
Status Status
Sort SortMode
Category Category
Category Categories
Max uint32
Offset uint32
UserID uint32
@ -86,13 +86,7 @@ func (p *TorrentParam) FromRequest(r *http.Request) {
p.FromDate = time.Now().AddDate(0, 0, -maxage).Format("2006-01-02")
}
var category Category
cat := r.URL.Query().Get("cat")
if cat != "" {
category.Parse(r.URL.Query().Get("cat"))
}
category.Parse(r.URL.Query().Get("c"))
categories := ParseCategories(r.URL.Query().Get("c"))
var sortMode SortMode
sortMode.Parse(r.URL.Query().Get("sort"))
@ -120,7 +114,7 @@ func (p *TorrentParam) FromRequest(r *http.Request) {
p.Order = ascending
p.Status = status
p.Sort = sortMode
p.Category = category
p.Category = categories
p.Language = language
p.MinSize = minSize
p.MaxSize = maxSize
@ -136,11 +130,16 @@ func (p *TorrentParam) FromRequest(r *http.Request) {
func (p *TorrentParam) ToFilterQuery() string {
// Don't set sub category unless main category is set
query := ""
if p.Category.IsMainSet() {
query += "category:" + strconv.FormatInt(int64(p.Category.Main), 10)
if p.Category.IsSubSet() {
query += " sub_category:" + strconv.FormatInt(int64(p.Category.Sub), 10)
if len(p.Category) > 0 {
conditionsOr := make([]string, len(p.Category))
for key, val := range p.Category {
if val.IsSubSet() {
conditionsOr[key] = "(category: " + strconv.FormatInt(int64(val.Main), 10) + " AND sub_category: " + strconv.FormatInt(int64(val.Sub), 10) + ")"
} else if val.IsMainSet() {
conditionsOr[key] = "(category: " + strconv.FormatInt(int64(val.Main), 10) + ")"
}
}
query += strings.Join(conditionsOr, " OR ")
}
if p.UserID != 0 {

16
database/postgres/where.go
Voir le fichier

@ -6,6 +6,7 @@ import (
"strings"
"github.com/NyaaPantsu/nyaa/common"
"github.com/NyaaPantsu/nyaa/config"
"github.com/NyaaPantsu/nyaa/model"
)
@ -13,11 +14,16 @@ import (
func searchParamToTorrentQuery(param *common.TorrentParam) (q sqlQuery) {
counter := 1
q.query = fmt.Sprintf("SELECT %s FROM %s ", torrentSelectColumnsFull, tableTorrents)
if param.Category.IsSet() {
q.query += fmt.Sprintf("WHERE category = $%d AND sub_category = $%d ", counter, counter+1)
q.params = append(q.params, param.Category.Main, param.Category.Sub)
counter += 2
if len(param.Category) > 0 {
conditionsOr := make([]string, len(param.Category))
for key, val := range param.Category {
conditionsOr[key] = fmt.Sprintf("(category = $%d AND sub_category = $%d)", counter, counter+1)
q.params = append(q.params, val.Main, val.Sub)
counter += 2
}
q.query += "WHERE " + strings.Join(conditionsOr, " OR ")
}
if counter > 1 {
q.query += "AND "
} else {
@ -94,7 +100,7 @@ func searchParamToTorrentQuery(param *common.TorrentParam) (q sqlQuery) {
sort = "completed"
case common.ID:
default:
sort = "torrent_id"
sort = config.Conf.Torrents.Order
}
q.query += fmt.Sprintf("ORDER BY %s ", sort)

12
router/modpanel.go
Voir le fichier

@ -167,9 +167,13 @@ func TorrentsListPanel(w http.ResponseWriter, r *http.Request) {
if err != nil {
messages.ImportFromError("errors", err)
}
category := ""
if len(searchParam.Category) > 0 {
category = searchParam.Category[0].String()
}
searchForm := searchForm{
SearchParam: searchParam,
Category: searchParam.Category.String(),
Category: category,
ShowItemsPerPage: true,
}
@ -492,9 +496,13 @@ func DeletedTorrentsModPanel(w http.ResponseWriter, r *http.Request) {
if err != nil {
messages.ImportFromError("errors", err)
}
category := ""
if len(searchParam.Category) > 0 {
category = searchParam.Category[0].String()
}
searchForm := searchForm{
SearchParam: searchParam,
Category: searchParam.Category.String(),
Category: category,
ShowItemsPerPage: true,
}

118
router/rss_handler.go
Voir le fichier

@ -139,12 +139,17 @@ func RSSTorznabHandler(w http.ResponseWriter, r *http.Request) {
T := publicSettings.GetTfuncFromRequest(r)
cat := categories.GetCategoriesSelect(true, true)
var categories []*nyaafeeds.RssCategoryTorznab
categories = append(categories, &nyaafeeds.RssCategoryTorznab{
ID: "5070",
Name: "Anime",
Description: "Anime",
})
var keys []string
for name := range cat {
keys = append(keys, name)
}
sort.Strings(keys)
last := -1
last := 0
for _, key := range keys {
if len(cat[key]) <= 2 {
categories = append(categories, &nyaafeeds.RssCategoryTorznab{
@ -208,13 +213,27 @@ func RSSTorznabHandler(w http.ResponseWriter, r *http.Request) {
feed := &nyaafeeds.RssFeed{
Title: title,
Xmlns: "http://torznab.com/schemas/2015/feed",
Link: config.WebAddress() + "/",
PubDate: createdAsTime.String(),
}
feed.Items = make([]*nyaafeeds.RssItem, len(torrents))
for i, torrent := range torrents {
torrentJSON := torrent.ToJSON()
filesNumber := ""
if len(torrentJSON.FileList) > 0 {
filesNumber = strconv.Itoa(len(torrentJSON.FileList))
}
seeders := ""
if torrentJSON.Seeders > 0 {
seeders = strconv.Itoa(int(torrentJSON.Seeders))
}
leechers := ""
if torrentJSON.Leechers > 0 {
leechers = strconv.Itoa(int(torrentJSON.Leechers))
}
feed.Items[i] = &nyaafeeds.RssItem{
Title: torrentJSON.Name,
Link: config.WebAddress() + "/download/" + torrentJSON.Hash,
@ -230,15 +249,35 @@ func RSSTorznabHandler(w http.ResponseWriter, r *http.Request) {
Length: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
Type: "application/x-bittorrent",
},
Torznab: &nyaafeeds.RssTorznab{
Xmlns: "http://torznab.com/schemas/2015/feed",
Size: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
Files: strconv.Itoa(len(torrentJSON.FileList)),
Grabs: strconv.Itoa(torrentJSON.Downloads),
Seeders: strconv.Itoa(int(torrentJSON.Seeders)),
Leechers: strconv.Itoa(int(torrentJSON.Leechers)),
Infohash: torrentJSON.Hash,
MagnetURL: string(torrentJSON.Magnet),
Torznab: []*nyaafeeds.RssTorznab{
&nyaafeeds.RssTorznab{
Name: "size",
Value: strconv.FormatUint(uint64(torrentJSON.Filesize), 10),
},
&nyaafeeds.RssTorznab{
Name: "files",
Value: filesNumber,
},
&nyaafeeds.RssTorznab{
Name: "grabs",
Value: strconv.Itoa(torrentJSON.Downloads),
},
&nyaafeeds.RssTorznab{
Name: "seeders",
Value: seeders,
},
&nyaafeeds.RssTorznab{
Name: "leechers",
Value: leechers,
},
&nyaafeeds.RssTorznab{
Name: "infohash",
Value: torrentJSON.Hash,
},
&nyaafeeds.RssTorznab{
Name: "magneturl",
Value: string(torrentJSON.Magnet),
},
},
}
}
@ -262,21 +301,37 @@ func getTorrentList(r *http.Request) (torrents []model.Torrent, createdAsTime ti
page := vars["page"]
userID := vars["id"]
cat := r.URL.Query().Get("cat")
offset := r.URL.Query().Get("offset")
pagenum := 1
if page == "" && offset != "" {
page = offset
offset := 0
if r.URL.Query().Get("offset") != "" {
offset, err = strconv.Atoi(html.EscapeString(r.URL.Query().Get("offset")))
if err != nil {
return
}
}
if page != "" {
createdAsTime = time.Now()
if len(torrents) > 0 {
createdAsTime = torrents[0].Date
}
title = "Nyaa Pantsu"
if config.IsSukebei() {
title = "Sukebei Pantsu"
}
pagenum := 1
if page == "" && offset > 0 { // first page for offset is 0
pagenum = offset + 1
} else if page != "" {
pagenum, err = strconv.Atoi(html.EscapeString(page))
if err != nil {
return
}
if pagenum <= 0 {
err = errors.New("Page number is invalid")
return
}
}
if pagenum <= 0 {
err = errors.New("Page number is invalid")
return
}
if userID != "" {
@ -299,22 +354,19 @@ func getTorrentList(r *http.Request) (torrents []model.Torrent, createdAsTime ti
if cat != "" {
query := r.URL.Query()
c, sub := nyaafeeds.ConvertToCat(cat)
query.Set("c", c+"_"+sub)
if cat == "5070" {
query.Set("c", "3_")
} else {
c := nyaafeeds.ConvertToCat(cat)
if c == "" {
return
}
query.Set("c", c)
}
r.URL.RawQuery = query.Encode()
}
_, torrents, err = search.SearchByQueryNoCount(r, pagenum)
createdAsTime = time.Now()
if len(torrents) > 0 {
createdAsTime = torrents[0].Date
}
title = "Nyaa Pantsu"
if config.IsSukebei() {
title = "Sukebei Pantsu"
}
return
}

10
router/search_handler.go
Voir le fichier

@ -1,11 +1,12 @@
package router
import (
"github.com/gorilla/mux"
"html"
"net/http"
"strconv"
"github.com/gorilla/mux"
"github.com/NyaaPantsu/nyaa/model"
"github.com/NyaaPantsu/nyaa/util/log"
msg "github.com/NyaaPantsu/nyaa/util/messages"
@ -47,11 +48,16 @@ func SearchHandler(w http.ResponseWriter, r *http.Request) {
commonVar.Navigation = navigation{int(nbTorrents), int(searchParam.Max), int(searchParam.Page), "search_page"}
// Convert back to strings for now.
// TODO Deprecate fully SearchParam and only use TorrentParam
category := ""
if len(searchParam.Category) > 0 {
category = searchParam.Category[0].String()
}
commonVar.Search = searchForm{
SearchParam: searchParam,
Category: searchParam.Category.String(),
Category: category,
ShowItemsPerPage: true,
}
htv := modelListVbs{commonVar, model.TorrentsToJSON(torrents), messages.GetAllErrors(), messages.GetAllInfos()}
err = searchTemplate.ExecuteTemplate(w, "index.html", htv)

51
util/feeds/convert.go
Voir le fichier

@ -1,25 +1,60 @@
package nyaafeeds
import (
"strconv"
"strings"
"github.com/NyaaPantsu/nyaa/util/categories"
)
// ConvertToCat : Convert a torznab cat to our cat
func ConvertToCat(cat string) (string, string) {
c := strings.Split(cat, "0000")
if len(c) < 2 {
return cat, ""
func ConvertToCat(cat string) string {
if cat == "" {
return ""
}
return c[0], c[1]
cats := strings.Split(cat, ",")
var returnCat []string
for _, val := range cats {
localeCat := convertCat(val)
if localeCat != "" {
returnCat = append(returnCat, localeCat)
}
}
return strings.Join(returnCat, ",")
}
func convertCat(cat string) string {
if len(cat) < 6 {
return ""
}
cI, _ := strconv.Atoi(cat[2:4])
subI, _ := strconv.Atoi(cat[4:6])
c := strconv.Itoa(cI)
sub := strconv.Itoa(subI)
if categories.CategoryExists(c + "_" + sub) {
return c + "_" + sub
}
return ""
}
// ConvertFromCat : Convert a cat to a torznab cat
func ConvertFromCat(category string) (cat string) {
c := strings.Split(category, "_")
if len(c) < 2 {
cat = c[0] + "0000"
if len(c[0]) < 2 {
c[0] = "0" + c[0]
}
if len(c) < 2 || c[1] == "" {
cat = "10" + c[0] + "00"
return
}
cat = c[0] + "0000" + c[1]
if len(c[1]) < 2 {
c[1] = "0" + c[1]
}
cat = "10" + c[0] + c[1]
return
}

35
util/feeds/rss.go
Voir le fichier

@ -16,6 +16,7 @@ import (
// private wrapper around the RssFeed which gives us the <rss>..</rss> xml
type rssFeedXML struct {
XMLName xml.Name `xml:"rss"`
Xmlns string `xml:"xmlns:torznab,attr,omitempty"`
Version string `xml:"version,attr"`
Encoding string `xml:"encoding,attr"`
Channel *RssFeed `xml:"channel,omitempty"`
@ -41,6 +42,7 @@ type RssTextInput struct {
type RssFeed struct {
XMLName xml.Name `xml:"channel"`
Xmlns string `xml:"-"`
Title string `xml:"title"` // required
Link string `xml:"link"` // required
Description string `xml:"description"` // required
@ -76,7 +78,7 @@ type RssItem struct {
PubDate string `xml:"pubDate,omitempty"` // created or updated
Source string `xml:"source,omitempty"`
Torrent *RssTorrent `xml:"torrent,omitempty"`
Torznab *RssTorznab `xml:"torznab,omitempty"`
Torznab []*RssTorznab
}
type RssCaps struct {
@ -101,7 +103,7 @@ type RssServer struct {
type RssLimits struct {
XMLName xml.Name `xml:"limits"`
Max string `xml:"limits,attr"`
Max string `xml:"max,attr"`
Default string `xml:"default,attr"`
}
@ -153,27 +155,9 @@ type RssTorrent struct {
}
type RssTorznab struct {
XMLName xml.Name `xml:"torznab"`
Xmlns string `xml:"xmlns,attr"`
Type string `xml:"type,omitempty"`
Size string `xml:"size,omitempty"`
Files string `xml:"files,omitempty"`
Grabs string `xml:"grabs,omitempty"`
Tvdbid string `xml:"tvdbid,omitempty"`
Rageid string `xml:"rageid,omitempty"`
Tvmazeid string `xml:"tvmazeid,omitempty"`
Imdb string `xml:"imdb,omitempty"`
BannerURL string `xml:"bannerurl,omitempty"`
Infohash string `xml:"infohash,omitempty"`
MagnetURL string `xml:"magneturl,omitempty"`
Seeders string `xml:"seeders,omitempty"`
Leechers string `xml:"leechers,omitempty"`
Peers string `xml:"peers,omitempty"`
SeedType string `xml:"seedtype,omitempty"`
MinimumRatio string `xml:"minimumratio,omitempty"`
MinimumSeedTime string `xml:"minimumseedtime,omitempty"`
DownloadVolumeFactor string `xml:"downloadvolumefactor,omitempty"`
UploadVolumeFactor string `xml:"uploadvolumefactor,omitempty"`
XMLName xml.Name `xml:"torznab:attr"`
Name string `xml:"name,attr"`
Value string `xml:"value,attr,omitempty"`
}
// RssCategory is a category for rss item
@ -261,10 +245,13 @@ func (r *Rss) FeedXml() interface{} {
// FeedXml : return an XML-ready object for an RssFeed object
func (r *RssFeed) FeedXml() interface{} {
if r.Xmlns != "" {
return &rssFeedXML{Version: "2.0", Encoding: "UTF-8", Channel: r, Xmlns: r.Xmlns}
}
return &rssFeedXML{Version: "2.0", Encoding: "UTF-8", Channel: r}
}
// FeedXml : return an XML-ready object for an RssFeed object
func (r *RssCaps) FeedXml() interface{} {
return &rssFeedXML{Version: "2.0", Encoding: "UTF-8", Caps: r}
return r
}

20
util/search/search.go
Voir le fichier

@ -133,9 +133,8 @@ func searchByQueryPostgres(r *http.Request, pagenum int, countAll bool, withUser
} else {
search.FromDate = time.Now().AddDate(0, 0, -maxage).Format("2006-01-02")
}
search.Category = common.ParseCategories(r.URL.Query().Get("c"))
search.Status.Parse(r.URL.Query().Get("s"))
search.Category.Parse(r.URL.Query().Get("c"))
search.Sort.Parse(r.URL.Query().Get("sort"))
search.MinSize.Parse(r.URL.Query().Get("minSize"))
search.MaxSize.Parse(r.URL.Query().Get("maxSize"))
@ -165,11 +164,16 @@ func searchByQueryPostgres(r *http.Request, pagenum int, countAll bool, withUser
Params: make([]interface{}, 0, 64),
}
conditions := make([]string, 0, 64)
if search.Category.Main != 0 {
conditions = append(conditions, "category = ?")
parameters.Params = append(parameters.Params, search.Category.Main)
if len(search.Category) > 0 {
conditionsOr := make([]string, len(search.Category))
for key, val := range search.Category {
conditionsOr[key] = "(category = ? AND sub_category = ?)"
parameters.Params = append(parameters.Params, val.Main)
parameters.Params = append(parameters.Params, val.Sub)
}
conditions = append(conditions, strings.Join(conditionsOr, " OR "))
}
if search.UserID != 0 {
conditions = append(conditions, "uploader = ?")
parameters.Params = append(parameters.Params, search.UserID)
@ -186,10 +190,6 @@ func searchByQueryPostgres(r *http.Request, pagenum int, countAll bool, withUser
conditions = append(conditions, "date <= ?")
parameters.Params = append(parameters.Params, search.ToDate)
}
if search.Category.Sub != 0 {
conditions = append(conditions, "sub_category = ?")
parameters.Params = append(parameters.Params, search.Category.Sub)
}
if search.Status != 0 {
if search.Status == common.FilterRemakes {
conditions = append(conditions, "status <> ?")