Albirew/nyaa-pantsu
Albirew
/
nyaa-pantsu
Archivé
1
0
Bifurcation 0

Merge branch 'dev' into filelist-fetching

Cette révision appartient à :
kilo 2018-01-10 12:46:27 +01:00 révisé par GitHub
révision 050d00dda3
Signature inconnue de Forgejo
ID de la clé GPG: 4AEE18F83AFDEB23
27 fichiers modifiés avec 3121 ajouts et 2188 suppressions

Voir le fichier

@ -54,6 +54,23 @@ scraper:
max_stat_scraping_frequency: 5
# MaxStatScrapingFrequencyUnknown : On-demand stat scraping for unknown torrents can only be called every X minutes
max_stat_scraping_frequency_unknown: 2
# Default config for upload
upload:
# The following token/accounts are used when an user wants to upload to another website
# but doesn't have his own API token.
anidex_api_token:
# TokyoTosho : Be aware that file upload doesn't work with the host (only url upload)
# so you need to enable users to download without ddos protection
tokyot_api_token:
# Nyaa.Si : we need a username and a password for the account upload, please use a difficult one!
nyaasi_username:
nyaasi_password:
# UploadsDisabled : Disable uploads for everyone except below
uploads_disabled: false
# AdminsAreStillAllowedTo : Enable admin torrent upload even if UploadsDisabled is true
admins_are_still_allowed_to: true
# TrustedUsersAreStillAllowedTo : Enable trusted users torrent upload even if UploadsDisabled is true
trusted_users_are_still_allowed_to: true
# Config by default for the cache
cache:
dialect: nop
@ -115,12 +132,6 @@ torrents:
storage_link: /download/%s
# TorrentCacheLink : Url of torrent site cache
cache_link: #http://anicache.com/torrent/%s.torrent
# UploadsDisabled : Disable uploads for everyone except below
uploads_disabled: false
# AdminsAreStillAllowedTo : Enable admin torrent upload even if UploadsDisabled is true
admins_are_still_allowed_to: true
# TrustedUsersAreStillAllowedTo : Enable trusted users torrent upload even if UploadsDisabled is true
trusted_users_are_still_allowed_to: true
trackers:
# Trackers : Default trackers supported
default:
@ -140,6 +151,22 @@ torrents:
# NeededTrackers : Array indexes of Trackers for needed tracker in a torrent file
needed:
- 0
- 12
dead:
- ://open.nyaatorrents.info:6544
- ://tracker.openbittorrent.com:80
- ://tracker.publicbt.com:80
- ://stats.anisource.net:2710
- ://exodus.desync.com
- ://open.demonii.com:1337
- ://tracker.istole.it:80
- ://tracker.ccc.de:80
- ://bt2.careland.com.cn:6969
- ://announce.torrentsmd.com:8080
- ://open.demonii.com:1337
- ://tracker.btcake.com
- ://tracker.prq.to
- ://bt.rghost.net
# TorrentOrder : Default sorting field for torrents
order: date
# TorrentSort : Default sorting order for torrents

Voir le fichier

@ -107,27 +107,28 @@ type I18nConfig struct {
// ScrapeConfig : Config struct for Scraping
type ScrapeConfig struct {
URL string `json:"scrape_url" yaml:"url,omitempty"`
Name string `json:"name" yaml:"name,omitempty"`
IntervalSeconds int64 `json:"interval" yaml:"interval,omitempty"`
URL string `json:"scrape_url" yaml:"url,omitempty"`
Name string `json:"name" yaml:"name,omitempty"`
IntervalSeconds int64 `json:"interval" yaml:"interval,omitempty"`
}
// ScraperConfig : Config struct for Scraper
type ScraperConfig struct {
Addr string `json:"bind" yaml:"addr,omitempty"`
NumWorkers int `json:"workers" yaml:"workers,omitempty"`
IntervalSeconds int64 `json:"default_interval" yaml:"default_interval,omitempty"`
Trackers []ScrapeConfig `json:"trackers" yaml:"trackers,omitempty"`
StatScrapingFrequency float64 `json:"stat_scraping_frequency" yaml:"stat_scraping_frequency,omitempty"`
StatScrapingFrequencyUnknown float64 `json:"stat_scraping_frequency_unknown" yaml:"stat_scraping_frequency_unknown,omitempty"`
MaxStatScrapingFrequency float64 `json:"max_stat_scraping_frequency" yaml:"max_stat_scraping_frequency,omitempty"`
MaxStatScrapingFrequencyUnknown float64 `json:"max_stat_scraping_frequency_unknown" yaml:"max_stat_scraping_frequency_unknown,omitempty"`
Addr string `json:"bind" yaml:"addr,omitempty"`
NumWorkers int `json:"workers" yaml:"workers,omitempty"`
IntervalSeconds int64 `json:"default_interval" yaml:"default_interval,omitempty"`
Trackers []ScrapeConfig `json:"trackers" yaml:"trackers,omitempty"`
StatScrapingFrequency float64 `json:"stat_scraping_frequency" yaml:"stat_scraping_frequency,omitempty"`
StatScrapingFrequencyUnknown float64 `json:"stat_scraping_frequency_unknown" yaml:"stat_scraping_frequency_unknown,omitempty"`
MaxStatScrapingFrequency float64 `json:"max_stat_scraping_frequency" yaml:"max_stat_scraping_frequency,omitempty"`
MaxStatScrapingFrequencyUnknown float64 `json:"max_stat_scraping_frequency_unknown" yaml:"max_stat_scraping_frequency_unknown,omitempty"`
}
// TrackersConfig ; Config struct for Trackers
type TrackersConfig struct {
Default ArrayString `yaml:"default,flow,omitempty"`
NeededTrackers []int `yaml:"needed,flow,omitempty"`
DeadTrackers ArrayString `yaml:"dead,flow,omitempty"`
}
// TorrentsConfig : Config struct for Torrents
@ -151,12 +152,13 @@ type TorrentsConfig struct {
// UploadConfig : Config struct for uploading torrents
type UploadConfig struct {
DefaultAnidexToken string `yaml:"anidex_api_token,omitempty"`
DefaultNyaasiToken string `yaml:"nyaasi_api_token,omitempty"`
DefaultTokyoTToken string `yaml:"tokyot_api_token,omitempty"`
UploadsDisabled bool `yaml:"uploads_disabled,omitempty"`
AdminsAreStillAllowedTo bool `yaml:"admins_are_still_allowed_to,omitempty"`
TrustedUsersAreStillAllowedTo bool `yaml:"trusted_users_are_still_allowed_to,omitempty"`
DefaultAnidexToken string `yaml:"anidex_api_token,omitempty"`
DefaultNyaasiUsername string `yaml:"nyaasi_api_username,omitempty"`
DefaultNyaasiPassword string `yaml:"nyaasi_api_password,omitempty"`
DefaultTokyoTToken string `yaml:"tokyot_api_token,omitempty"`
UploadsDisabled bool `yaml:"uploads_disabled,omitempty"`
AdminsAreStillAllowedTo bool `yaml:"admins_are_still_allowed_to,omitempty"`
TrustedUsersAreStillAllowedTo bool `yaml:"trusted_users_are_still_allowed_to,omitempty"`
}
// UsersConfig : Config struct for Users
@ -225,9 +227,9 @@ type ModelsConfig struct {
}
type DefaultThemeConfig struct {
Theme string `yaml:"theme,omitempty"`
Dark string `yaml:"dark,omitempty"`
Forced string `yaml:"forced,omitempty"`
Theme string `yaml:"theme,omitempty"`
Dark string `yaml:"dark,omitempty"`
Forced string `yaml:"forced,omitempty"`
}
// SearchConfig : Config struct for search
@ -275,3 +277,14 @@ func (ty TagTypes) Get(tagType string) TagType {
}
return TagType{}
}
// GetDefault returns the first tracker from the needed ones
func (tc TrackersConfig) GetDefault() string {
if len(tc.NeededTrackers) > 0 {
return tc.Default[tc.NeededTrackers[0]]
}
if len(tc.Default) > 0 {
return tc.Default[0]
}
return ""
}

Voir le fichier

@ -31,16 +31,16 @@ func DownloadTorrent(c *gin.Context) {
templates.Render(c, "errors/torrent_file_missing.jet.html", variables)
return
}
if c.Query("js_query") != "" {
exists := true
generating := false
if len(config.Get().Torrents.FileStorage) == 0 {
exists = false
} else {
Openfile, err := os.Open(fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, hash))
defer Openfile.Close()
defer Openfile.Close()
if err != nil {
exists = false
generating = true
@ -52,14 +52,11 @@ func DownloadTorrent(c *gin.Context) {
trackers = torrent.GetTrackersArray()
}
magnet := format.InfoHashToMagnet(strings.TrimSpace(torrent.Hash), torrent.Name, trackers...)
if upload.GenerateTorrent(magnet) != nil {
//Error during the generation
generating = false
}
go upload.GenerateTorrent(magnet)
}
}
c.JSON(200, gin.H{ // Better to use gin for that, less code
"exists": exists,
"exists": exists,
"generating": generating,
})
return
@ -82,7 +79,7 @@ func DownloadTorrent(c *gin.Context) {
}
//Check if file exists and open
Openfile, err := os.Open(fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, hash))
Openfile, err := os.Open(torrent.GetPath())
if err != nil {
//File not found, send 404
variables := templates.Commonvariables(c)
@ -94,9 +91,7 @@ func DownloadTorrent(c *gin.Context) {
}
magnet := format.InfoHashToMagnet(strings.TrimSpace(torrent.Hash), torrent.Name, trackers...)
variables.Set("magnet", magnet)
if upload.GenerateTorrent(magnet) != nil {
messages.AddError("errors", "Could not generate torrent file")
}
go upload.GenerateTorrent(magnet)
templates.Render(c, "errors/torrent_file_missing.jet.html", variables)
return
}

Voir le fichier

@ -4,4 +4,5 @@ import "github.com/NyaaPantsu/nyaa/controllers/router"
func init() {
router.Get().Any("/upload", UploadHandler)
router.Get().Any("/upload/status/:id", multiUploadStatus)
}

Voir le fichier

@ -2,6 +2,7 @@ package uploadController
import (
"errors"
"fmt"
"net/http"
"strconv"
@ -9,13 +10,14 @@ import (
"github.com/NyaaPantsu/nyaa/models"
"github.com/NyaaPantsu/nyaa/models/torrents"
"github.com/NyaaPantsu/nyaa/templates"
"github.com/NyaaPantsu/nyaa/utils/cache"
"github.com/NyaaPantsu/nyaa/utils/captcha"
"github.com/NyaaPantsu/nyaa/utils/log"
msg "github.com/NyaaPantsu/nyaa/utils/messages"
"github.com/NyaaPantsu/nyaa/utils/publicSettings"
"github.com/NyaaPantsu/nyaa/utils/upload"
"github.com/NyaaPantsu/nyaa/utils/validator/torrent"
"github.com/gin-gonic/gin"
"github.com/NyaaPantsu/nyaa/utils/log"
)
// UploadHandler : Main Controller for uploading a torrent
@ -67,10 +69,53 @@ func UploadPostHandler(c *gin.Context) {
messages.AddError("errors", err.Error())
}
AnidexUpload := false
NyaaSiUpload := false
TokyoToshoUpload := false
if c.PostForm("anidex_api") != "" || c.PostForm("anidex_upload") == "true" {
AnidexUpload = true
}
if c.PostForm("nyaasi_api") != "" || c.PostForm("nyaasi_upload") == "true" {
NyaaSiUpload = true
}
if c.PostForm("tokyot_api") != "" || c.PostForm("tokyot_upload") == "true" {
TokyoToshoUpload = true
}
if !messages.HasErrors() {
// add to db and redirect
// add to db
torrent, err := torrents.Create(user, &uploadForm)
log.CheckErrorWithMessage(err, "ERROR_TORRENT_CREATED: Error while creating entry in db")
if AnidexUpload || NyaaSiUpload || TokyoToshoUpload {
go func(anidexApiKey string, anidexFormCategory string, anidexFormLang string, nyaasiUsername string, nyaasiPassword string, toshoApiKey string) {
err := upload.GotFile(torrent)
if err != nil {
log.CheckError(err)
return
}
// User wants to upload to other websites too
if AnidexUpload {
go upload.ToAnidex(torrent, anidexApiKey, anidexFormCategory, anidexFormLang)
}
if NyaaSiUpload {
go upload.ToNyaasi(nyaasiUsername, nyaasiPassword, torrent)
}
if TokyoToshoUpload {
go upload.ToTTosho(toshoApiKey, torrent)
}
}(c.PostForm("anidex_api"), c.PostForm("anidex_form_category"), c.PostForm("anidex_form_lang"), c.PostForm("nyaasi_username"), c.PostForm("nyaasi_password"), c.PostForm("tokyot_api"))
// After that, we redirect to the page for upload status
url := fmt.Sprintf("/upload/status/%d", torrent.ID)
c.Redirect(302, url)
return
}
// We don't need it to be synchronous since the generation can be left in a background process
go upload.GotFile(torrent)
url := "/view/" + strconv.FormatUint(uint64(torrent.ID), 10)
c.Redirect(302, url+"?success")
}
@ -88,3 +133,27 @@ func UploadGetHandler(c *gin.Context) {
}
templates.Form(c, "site/torrents/upload.jet.html", uploadForm)
}
// multiUploadStatus : controller to show the multi upload status
func multiUploadStatus(c *gin.Context) {
id := c.Param("id")
if id == "" {
c.AbortWithStatus(http.StatusNotFound)
return
}
if found, ok := cache.C.Get("tstatus_" + id); ok {
uploadMultiple := found.(upload.MultipleForm)
// if ?json we print the json format
if _, ok = c.GetQuery("json"); ok {
c.JSON(http.StatusOK, uploadMultiple)
return
}
// else we send the upload multiple form (support of manual F5)
variables := templates.Commonvariables(c)
variables.Set("UploadMultiple", uploadMultiple)
templates.Render(c, "site/torrents/upload_multiple.jet.html", variables)
} else {
// here it means the upload status is already flushed from memory
c.AbortWithStatus(http.StatusNotFound)
}
}

Voir le fichier

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"html/template"
"os"
"path/filepath"
"reflect"
"strconv"
@ -166,11 +167,22 @@ func (t *Torrent) IsDeleted() bool {
return t.DeletedAt != nil
}
// IsAnon : Return if a torrent is displayed as anon
// Be aware, it doesn't mean that the owner is anonymous!
func (t *Torrent) IsAnon() bool {
return t.Hidden || t.UploaderID == 0
}
// GetDescriptiveTags : Return the descriptive tags
func (t *Torrent) GetDescriptiveTags() string {
return t.AcceptedTags
}
// GetPath : Helpers to get the path to the torrent file
func (t *Torrent) GetPath() string {
return fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, t.Hash)
}
// AddToESIndex : Adds a torrent to Elastic Search
func (t Torrent) AddToESIndex(client *elastic.Client) error {
ctx := context.Background()
@ -224,7 +236,7 @@ func (t *Torrent) ParseTrackers(trackers []string) {
}
}
trackers = tempTrackers
v["tr"] = trackers
t.Trackers = v.Encode()
}
@ -349,23 +361,14 @@ func (t *Torrent) ToJSON() TorrentJSON {
} else if t.OldUploader != "" {
uploader = t.OldUploader
}
torrentlink := ""
if len(config.Get().Torrents.CacheLink) > 0 { // Only use torrent cache if set, don't check id since better to have all .torrent
if config.IsSukebei() {
torrentlink = "" // torrent cache doesn't have sukebei torrents
} else {
torrentlink = fmt.Sprintf(config.Get().Torrents.CacheLink, t.Hash)
}
} else if len(config.Get().Torrents.StorageLink) > 0 { // Only use own .torrent if storage set
torrentlink = fmt.Sprintf(config.Get().Torrents.StorageLink, t.Hash)
}
scrape := Scrape{}
if t.Scrape != nil {
scrape = *t.Scrape
}
statsObsolete := []bool{false, false}
if scrape.LastScrape.IsZero() || (scrape.Seeders == 0 && scrape.Leechers == 0 && scrape.Completed == 0) {
statsObsolete[0] = true
//The displayed stats are obsolete, S/D/L will show "Unknown"
@ -374,7 +377,7 @@ func (t *Torrent) ToJSON() TorrentJSON {
statsObsolete[1] = true
//The stats need to be refreshed, either because they are valid and older than one month (not that reliable) OR if they are unknown but have been scraped 1h (or more) ago
}
t.ParseLanguages()
res := TorrentJSON{
ID: t.ID,
@ -394,7 +397,7 @@ func (t *Torrent) ToJSON() TorrentJSON {
WebsiteLink: sanitize.Safe(t.WebsiteLink),
Languages: t.Languages,
Magnet: template.URL(magnet),
TorrentLink: sanitize.Safe(torrentlink),
TorrentLink: sanitize.Safe(t.Download()),
Leechers: scrape.Leechers,
Seeders: scrape.Seeders,
Completed: scrape.Completed,
@ -541,11 +544,25 @@ func (t *Torrent) DeleteTags() {
}
}
func contains(s []string, e string) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
// Download generate a download link for a torrent
func (t *Torrent) Download() (torrentlink string) {
if len(config.Get().Torrents.CacheLink) > 0 { // Only use torrent cache if set, don't check id since better to have all .torrent
if !config.IsSukebei() { // torrent cache doesn't have sukebei torrents
torrentlink = fmt.Sprintf(config.Get().Torrents.CacheLink, t.Hash)
}
return
}
if len(config.Get().Torrents.StorageLink) > 0 { // Only use own .torrent if storage set
torrentlink = fmt.Sprintf(config.Get().Torrents.StorageLink, t.Hash)
}
return
}
func contains(s []string, e string) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}

Voir le fichier

@ -70,6 +70,7 @@ a:hover, .h-nav a:hover, .h-nav:active a:hover {
.upload-form-table .checkbox-container+input {
width: 385px;1
}
.upload-form-table .table-checkboxes {
padding: 3px 0!important;
width: 100%!important;
@ -677,7 +678,7 @@ span.tag {
}
.upload-form-table {
width: auto;
width: 542px;
}
.upload-form-table .table-input-label label::after {
content: ':';

Voir le fichier

@ -334,7 +334,11 @@ span.tag {
.upload-form-table .checkbox-container {
border-color: #0c0d0e;
background: #333438;
background: #28282b;
}
#anidex-upload-info > div input[type="text"], #anidex-upload-info > div select {
background-color: #28282b;
}
.torrent-info-row .tr-se span, .torrent-info-row .tr-le span, .torrent-info-row .tr-dl span {

Voir le fichier

@ -59,8 +59,10 @@ function parseAllDates() {
var list = document.getElementsByClassName("date-short")
for(var i = 0; i < list.length; i++) {
var e = list[i]
e.innerText = new Date(e.title).toLocaleString(lang, ymdOpt)
e.title = new Date(e.title).toLocaleString(lang)
var newDate = new Date(e.title)
if(isNaN(newDate.valueOf())) continue
e.innerText = newDate.toLocaleString(lang, ymdOpt)
e.title = newDate.toLocaleString(lang)
}
var list = document.getElementsByClassName("date-full")

Voir le fichier

@ -1,8 +1,8 @@
{{ extends "layouts/index_admin" }}
{{block title()}}{{ T("moderation_guidelines") }}{{end}}
{{ block content_body()}}
<div class="results box">
<h1>{{ T("moderation_guidelines") }}</h1>
<img src="https://www.themarysue.com/wp-content/uploads/2016/09/harassment-guide.png">
</div>
{{end}}
{{ extends "layouts/index_admin" }}
{{block title()}}{{ T("moderation_guidelines") }}{{end}}
{{ block content_body()}}
<div class="results box">
<h1>{{ T("moderation_guidelines") }}</h1>
<img src="https://www.themarysue.com/wp-content/uploads/2016/09/harassment-guide.png">
</div>
{{end}}

Voir le fichier

@ -66,6 +66,90 @@
<td class="table-input-label"><label for="desc">{{ T("torrent_description")}}</label></td>
<td class="table-input markdown-container"><textarea name="desc" id="desc" class="form-input up-input" style="height: 10rem;">{{Form.Description}}</textarea></td>
</tr>
<tr>
<td class="table-input-label">{{ T("upload_to") }} Anidex, Nyaa.si, TokyoTosho:</td>
<td class="table-input">
<details id="anidex-upload-info">
<summary class="form-input">{{ T("upload_to_other") }}</summary>
<div class="form-input">
<p>{{ T("upload_to") }} <b>Anidex</b>:</p>
<div>
<div class="checkbox-container"><input type="checkbox" value="true" name="anidex_upload" id="anidex_upload" class="form-torrent" {{if User.AnidexAPIToken != ""}}checked{{end}} /></div>
<input name="anidex_api" id="anidex_api" placeholder="Anidex API token" class="form-input" type="text" value="{{User.AnidexAPIToken}}" {{if User.AnidexAPIToken != ""}}disabled{{end}}/>
<br/>
<select class="form-input" name="anidex_form_category">
<option>{{ T("choose_category") }}</option>
<option value="1" style="background-color: #9f9;">Anime - Sub</option>
<option value="2" style="background-color: #9c9;">Anime - Raw</option>
<option value="3" style="background-color: #3f3;">Anime - Dub</option>
<option value="4" style="background-color: #9ff;">LA - Sub</option>
<option value="5" style="background-color: #9cc;">LA - Raw</option>
<option value="6" style="background-color: #ff6;">Light Novel</option>
<option value="7" style="background-color: #f9f;">Manga - Translated</option>
<option value="8" style="background-color: #c9c;">Manga - Raw</option>
<option value="9" style="background-color: #f99;">Audio - Lossy</option>
<option value="10" style="background-color: #c99;">Audio - Lossless</option>
<option value="11" style="background-color: #f66;">Video</option>
<option value="12" style="background-color: #ff9;">Games</option>
<option value="13" style="background-color: #cc9;">Applications</option>
<option value="14" style="background-color: #99c;">Pictures</option>
{{if Sukebei()}}<option value="15" style="background-color: #99f;">Adult Video</option>{{end}}
<option value="16" style="background-color: #999;">Other</option>
</select>
<select class="form-input" name="anidex_form_lang">
<option>{{ T("choose_language") }}</option>
<option value="1">English</option>
<option value="2">Japanese</option>
<option value="3">Polish</option>
<option value="4">Serbo-Croatian</option>
<option value="5">Dutch</option>
<option value="6">Italian</option>
<option value="7">Russian</option>
<option value="8">German</option>
<option value="9">Hungarian</option>
<option value="10">French</option>
<option value="11">Finnish</option>
<option value="12">Vietnamese</option>
<option value="13">Greek</option>
<option value="14">Bulgarian</option>
<option value="15">Spanish</option>
<option value="16">Portuguese (Brasil)</option>
<option value="17">Portuguese (Portugal)</option>
<option value="18">Swedish</option>
<option value="19">Arabic</option>
<option value="20">Danish</option>
<option value="21">Chinese (Simplified)</option>
<option value="22">Bengali</option>
<option value="23">Romanian</option>
<option value="24">Czech</option>
<option value="25">Mongolian</option>
<option value="26">Turkish</option>
<option value="27">Indonesian</option>
<option value="28">Korean</option>
<option value="29">Spanish (LATAM)</option>
<option value="30">Persian</option>
<option value="31">Malaysian</option>
</select>
</div>
<p>{{ T("upload_to") }} <b>Nyaa.si</b>:</p>
<div>
<div class="checkbox-container"><input type="checkbox" value="true" name="nyaasi_upload" id="nyaasi_upload" class="form-torrent"/></div>
<input name="nyaasi_username" id="nyaasi_username" placeholder="Nyaa.si Username" class="form-input up-input" type="text" value="" />
<br/>
<input name="nyaasi_password" id="nyaasi_password" placeholder="Nyaa.si Password" class="form-input up-input" type="password" value="" />
</div>
<p>{{ T("upload_to") }} <b>TokyoTosho</b>:</p>
<div>
<div class="checkbox-container"><input type="checkbox" value="true" name="tokyot_upload" id="tokyot_upload" {{if User.TokyoTAPIToken != ""}}checked{{end}} class="form-torrent"/></div>
<input name="tokyot_api" id="tokyot_api" placeholder="TokyoTosho API token" class="form-input up-input" type="text" value="{{User.TokyoTAPIToken }}" {{if User.AnidexAPIToken != ""}}disabled{{end}}/>
</div>
</div>
</details>
</td>
</tr>
<tr>
<td class="table-input-label">{{ T("torrent_tags")}}:</td>
<td class="table-input">

Voir le fichier

@ -2,20 +2,30 @@
{{block title()}}{{ T("upload")}}{{end}}
{{block content_body()}}
<div style="text-align: left;" class="box">
<h1 style="text-align:center;margin-top; 4px;">Upload status</h1>
<h1 style="text-align:center;margin-top: 4px;">{{ T("upload_status") }}</h1>
<table class="multiple-upload">
<tbody>
<tr><td colspan="4"><h3>Nyaa Pantsu upload status</h3></td></tr>
<tr class="upload-status"><td class="upload-site-name">Nyaa Pantsu</td><td>Status:</td><td class="icon-finished finished"></td><td class="upload-progress finished">Finished</td></tr>
<tr><td colspan="4" class="uploaded-url"><input type="text" class="form-input" placeholder="http://" value="http://nyaa.pantsu.cat/view/4556456" disabled></td></tr>
<tr><td colspan="4"><h3>Nyaa Pantsu {{ T("upload_status") }}</h3></td></tr>
<tr class="upload-status"><td class="upload-site-name">Nyaa Pantsu</td><td>{{ T("status") }}:</td><td class="icon-finished finished"></td><td class="upload-progress finished">{{ T("finished") }}</td></tr>
<tr><td colspan="4" class="uploaded-url"><a href="/view/{{UploadMultiple.PantsuID}}" target="_blank"><input type="text" class="form-input" value="{{if Sukebei()}}{{Config.WebAddress.Sukebei}}{{else}}{{Config.WebAddress.Nyaa}}{{end}}/view/{{UploadMultiple.PantsuID}}" disabled></a></td></tr>
<tr><td colspan="4"><h3>Nyaa.si upload status</h3></td></tr>
<tr class="upload-status"><td class="upload-site-name">Nyaa.si</td><td>Status:</td><td class="icon-pending pending"></td><td class="upload-progress pending">Pending</td></tr>
<tr><td colspan="4" class="uploaded-url"><input type="text" class="form-input" placeholder="Uploading..." disabled></td></tr>
{{ if UploadMultiple.Nyaasi.Status != 0 }}
<tr><td colspan="4"><h3>Nyaa.si {{ T("upload_status") }}</h3></td></tr>
<tr class="upload-status nyaasi"><td class="upload-site-name">Nyaa.si</td><td>{{ T("status") }}:</td><td class="statusicon icon-{{ if UploadMultiple.Nyaasi.Status == 1}}pending pending{{else if UploadMultiple.Nyaasi.Status == 2}}error error{{else}}finished finished{{end}}"></td><td class="upload-progress {{ if UploadMultiple.Nyaasi.Status == 1}}pending{{else if UploadMultiple.Nyaasi.Status == 2}}error{{else}}finished{{end}}">{{ if UploadMultiple.Nyaasi.Status == 1}}{{ T("pending") }}{{else if UploadMultiple.Nyaasi.Status == 2}}{{ T("error") }}{{else}}{{ T("finished") }}{{end}}</td></tr>
<tr><td colspan="4" class="uploaded-url"><input type="text"id="nyaasiMess" class="form-input" value="{{UploadMultiple.Nyaasi.Message}}" disabled></td></tr>
{{end}}
<tr><td colspan="4"><h3>Anidex upload status</h3></td></tr>
<tr class="upload-status"><td class="upload-site-name">Anidex</td><td>Status:</td><td class="icon-error error"></td><td class="upload-progress error">Error</td></tr>
<tr><td colspan="4" class="uploaded-url"><input type="text" class="form-input" placeholder="http://" value="Error: XXXX" disabled></td></tr>
{{ if UploadMultiple.Anidex.Status != 0 }}
<tr><td colspan="4"><h3>Anidex {{ T("upload_status") }}</h3></td></tr>
<tr class="upload-status anidex"><td class="upload-site-name">Anidex</td><td>{{ T("status") }}:</td><td class="statusicon icon-{{ if UploadMultiple.Anidex.Status == 1}}pending pending{{else if UploadMultiple.Anidex.Status == 2}}error error{{else}}finished finished{{end}}"></td><td class="upload-progress {{ if UploadMultiple.Anidex.Status == 1}}pending{{else if UploadMultiple.Anidex.Status == 2}}error{{else}}finished{{end}}">{{ if UploadMultiple.Anidex.Status == 1}}{{ T("pending") }}{{else if UploadMultiple.Anidex.Status == 2}}{{ T("error") }}{{else}}{{ T("finished") }}{{end}}</td></tr>
<tr><td colspan="4" class="uploaded-url"><input type="text" id="anidexMess" class="form-input" value="{{UploadMultiple.Anidex.Message}}" disabled></td></tr>
{{end}}
{{ if UploadMultiple.TTosho.Status != 0 }}
<tr><td colspan="4"><h3>Tokyo Tosho {{ T("upload_status") }}</h3></td></tr>
<tr class="upload-status ttosho"><td class="upload-site-name">Tokyo Tosho</td><td>{{ T("status") }}:</td><td class="statusicon icon-{{ if UploadMultiple.TTosho.Status == 1}}pending pending{{else if UploadMultiple.TTosho.Status == 2}}error error{{else}}finished finished{{end}}"></td><td class="upload-progress {{ if UploadMultiple.TTosho.Status == 1}}pending{{else if UploadMultiple.TTosho.Status == 2}}error{{else}}finished{{end}}">{{ if UploadMultiple.TTosho.Status == 1}}{{ T("pending") }}{{else if UploadMultiple.TTosho.Status == 2}}{{ T("error") }}{{else}}{{ T("finished") }}{{end}}</td></tr>
<tr><td colspan="4" class="uploaded-url"><input type="text" id="ttoshoMess" class="form-input" value="{{UploadMultiple.TTosho.Message}}" disabled></td></tr>
{{end}}
</tbody>
</table>
@ -24,4 +34,38 @@
{{ block footer_js()}}
<script type="text/javascript" src="/js/translation.js?v={{ Config.Version}}{{ Config.Build }}"></script>
<script type="text/javascript" src="/js/upload.js?v={{ Config.Version}}{{ Config.Build }}"></script>
<script type="text/javascript">
var multiUploadStatus = ["", "pending", "error", "finished"]
T.Add("pending", "{{ T("pending") }}")
T.Add("error", "{{ T("error") }}")
T.Add("finished", "{{ T("finished") }}")
function checkStatus() {
Query.Get(window.location.href+"?json", function(data) {
count = 3
if (data.anidex.status > 0) {
document.getElementById("anidexMess").value = data.anidex.message
document.querySelector("tr.upload-status.anidex .statusicon").className = "statusicon icon-"+multiUploadStatus[data.anidex.status]+" "+multiUploadStatus[data.anidex.status]
document.querySelector("tr.upload-status.anidex .upload-progress").className = "upload-progress "+multiUploadStatus[data.anidex.status]
document.querySelector("tr.upload-status.anidex .upload-progress").textContent = T.r(multiUploadStatus[data.anidex.status])
}
if (data.anidex.status != 1) count--
if (data.nyaasi.status > 0) {
document.getElementById("nyaasiMess").value = data.nyaasi.message
document.querySelector("tr.upload-status.nyaasi .statusicon").className = "statusicon icon-"+multiUploadStatus[data.nyaasi.status]+" "+multiUploadStatus[data.nyaasi.status]
document.querySelector("tr.upload-status.nyaasi .upload-progress").className = "upload-progress "+multiUploadStatus[data.nyaasi.status]
document.querySelector("tr.upload-status.nyaasi .upload-progress").textContent = T.r(multiUploadStatus[data.anidex.status])
}
if (data.anidex.status != 1) count--
if (data.ttosho.status > 0) {
document.getElementById("ttoshoMess").value = data.ttosho.message
document.querySelector("tr.upload-status.nyaasi .statusicon").className = "statusicon icon-"+multiUploadStatus[data.ttosho.status]+" "+multiUploadStatus[data.ttosho.status]
document.querySelector("tr.upload-status.nyaasi .upload-progress").className = "upload-progress "+multiUploadStatus[data.ttosho.status]
document.querySelector("tr.upload-status.nyaasi .upload-progress").textContent = T.r(multiUploadStatus[data.anidex.status])
}
if (data.anidex.status != 1) count--
if (count > 0) setTimeout(checkStatus, 5000)
})
}
checkStatus()
</script>
{{end}}

Voir le fichier

@ -9,6 +9,7 @@ import (
"path"
"testing"
"github.com/NyaaPantsu/nyaa/utils/upload"
"github.com/NyaaPantsu/nyaa/utils/validator/announcement"
"strings"
@ -213,6 +214,10 @@ func walkDirTest(dir string, t *testing.T) {
variables.Set("Form", models.Tags{*fakeTag, *fakeTag, *fakeTag})
return variables
},
"upload_multiple.jet.html": func(variables jet.VarMap) jet.VarMap {
variables.Set("UploadMultiple", upload.MultipleForm{})
return variables
},
}
fmt.Printf("\nTesting Folder: %s\n", dir)

Voir le fichier

@ -77,6 +77,16 @@
* + read_notifications_cleared
* + notifications_read
* + mark_notifications_as_read
## 2017/10/30
* + status
* + upload_status
* + error
* + pending
* + finished
* + upload_to
* + upload_to_other
* + choose_category
* + choose_language
## 2017/10/31
* + followers
* + comment_markdown_notice

Voir le fichier

@ -2279,6 +2279,42 @@
"id": "status",
"translation": "Status"
},
{
"id": "upload_status",
"translation": "Upload status"
},
{
"id": "pending",
"translation": "Pending"
},
{
"id": "error",
"translation": "Error"
},
{
"id": "finished",
"translation": "Finished"
},
{
"id": "upload_to",
"translation": "Upload to"
},
{
"id": "upload_to_other",
"translation": "Upload to other websites"
},
{
"id": "upload_to_other",
"translation": "Upload to other websites"
},
{
"id":"choose_category",
"translation": "Choose a category (required)"
},
{
"id":"choose_language",
"translation":"Choose a language (required)"
},
{
"id": "event",
"translation": "Event"

Fichier diff supprimé car celui-ci est trop grand Voir la Diff

Voir le fichier

@ -2283,6 +2283,38 @@
"id": "status",
"translation": "Statut"
},
{
"id": "upload_status",
"translation": "Statut de l'upload"
},
{
"id": "pending",
"translation": "En attente"
},
{
"id": "error",
"translation": "Erreur"
},
{
"id": "finished",
"translation": "Terminé"
},
{
"id": "upload_to",
"translation": "Uploader vers"
},
{
"id": "upload_to_other",
"translation": "Uploader vers d'autres sites"
},
{
"id": "choose_category",
"translation": "Choisir une catégorie (requis)"
},
{
"id": "choose_language",
"translation": "Choisir une langue (requis)"
},
{
"id": "event",
"translation": "Événement"

95
utils/upload/categories.go Fichier normal
Voir le fichier

@ -0,0 +1,95 @@
package upload
import (
"fmt"
"github.com/NyaaPantsu/nyaa/config"
"github.com/NyaaPantsu/nyaa/models"
)
// Convert automatically our sukebei cats to platform specified Hentai cats
var sukebeiCategories = []map[string]string{
ttosho: {
"1_1": "12",
"1_2": "12",
"1_3": "14",
"1_4": "13",
"1_5": "4",
"2_1": "4",
"2_2": "15",
},
nyaasi: {
"1_1": "1_1",
"1_2": "1_2",
"1_3": "1_3",
"1_4": "1_4",
"1_5": "1_5",
"2_1": "2_1",
"2_2": "2_2",
},
}
var normalCategories = []map[string]string{
ttosho: {
"3_12": "1",
"3_5": "1",
"3_13": "10",
"3_6": "7",
"2_3": "2",
"2_4": "2",
"4_7": "3",
"4_8": "7",
"4_14": "10",
"5_9": "8",
"5_10": "8",
"5_18": "10",
"5_11": "7",
"6_15": "5",
"6_16": "5",
"1_1": "5",
"1_2": "5",
},
nyaasi: {
"3_12": "1_1",
"3_5": "1_2",
"3_13": "1_3",
"3_6": "1_4",
"2_3": "2_1",
"2_4": "2_2",
"4_7": "3_1",
"4_8": "3_4",
"4_14": "3_3",
"5_9": "4_1",
"5_10": "4_2",
"5_18": "4_3",
"5_11": "4_4",
"6_15": "5_1",
"6_16": "5_2",
"1_1": "6_1",
"1_2": "6_2",
},
}
// Category returns the category converted from nyaa one to tosho one
func Category(platform int, t *models.Torrent) string {
cat := fmt.Sprintf("%d_%d", t.Category, t.SubCategory)
// if we are in sukebei, there are some categories
if config.IsSukebei() {
// check that platform exist in our map for sukebei categories
if platform < len(sukebeiCategories) {
// return the remaped category if it exists
if val, ok := sukebeiCategories[platform][cat]; ok {
return val
}
}
}
// check that platform exist in our map
if platform >= len(normalCategories) {
return ""
}
// return the remaped category if it exists
if val, ok := normalCategories[platform][cat]; ok {
return val
}
return ""
}

Voir le fichier

@ -0,0 +1,38 @@
package upload
import (
"testing"
"github.com/NyaaPantsu/nyaa/config"
"github.com/NyaaPantsu/nyaa/models"
"github.com/stretchr/testify/assert"
)
func TestCategory(t *testing.T) {
assert := assert.New(t)
dummyTorrent := &models.Torrent{Category: 1, SubCategory: 1}
tests := []struct {
torrent *models.Torrent
platform int
sukebei bool
expected string
}{
{dummyTorrent, anidex, false, ""},
{dummyTorrent, ttosho, false, "5"},
{dummyTorrent, ttosho, true, "12"},
{dummyTorrent, nyaasi, false, "6_1"},
{dummyTorrent, nyaasi, true, "1_1"},
{dummyTorrent, 20, true, ""},
{&models.Torrent{Category: 33, SubCategory: 33}, nyaasi, true, ""},
}
for _, test := range tests {
if test.sukebei {
// workaround to make the function believe we are in sukebei
config.Get().Models.TorrentsTableName = "sukebei_torrents"
} else {
config.Get().Models.TorrentsTableName = "torrents"
}
assert.Equal(test.expected, Category(test.platform, test.torrent))
}
}

6
utils/upload/errors.go Fichier normal
Voir le fichier

@ -0,0 +1,6 @@
package upload
import "errors"
var errPending = errors.New("Magnet being generated already")
var errConfig = errors.New("Magnet Empty or FileStorage not configured")

Voir le fichier

@ -1,12 +1,15 @@
package upload
import (
"strconv"
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/NyaaPantsu/nyaa/config"
"github.com/NyaaPantsu/nyaa/models"
"github.com/NyaaPantsu/nyaa/utils/format"
"github.com/NyaaPantsu/nyaa/utils/log"
"github.com/anacrolix/dht"
"github.com/anacrolix/torrent"
@ -41,12 +44,12 @@ func GenerateTorrent(magnet string) error {
}
}
if magnet == "" || len(config.Get().Torrents.FileStorage) == 0 {
return errors.New("Magnet Empty or FileStorage not configured")
return errConfig
}
if len(queue) > 0 {
for _, m := range queue {
if m == magnet {
return errors.New("Magnet being generated already")
return errPending
}
}
}
@ -57,28 +60,63 @@ func GenerateTorrent(magnet string) error {
log.Errorf("error adding magnet to client: %s", err)
return err
}
go func() {
<-t.GotInfo()
mi := t.Metainfo()
t.Drop()
file := fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, t.InfoHash().String())
f, err := os.Create(file)
if err != nil {
log.Errorf("error creating torrent metainfo file: %s", err)
return
<-t.GotInfo()
mi := t.Metainfo()
t.Drop()
file := fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, t.InfoHash().String())
f, err := os.Create(file)
if err != nil {
log.Errorf("error creating torrent metainfo file: %s", err)
return err
}
defer f.Close()
defaultTracker := config.Get().Torrents.Trackers.GetDefault()
if defaultTracker != "" {
mi.Announce = defaultTracker
}
err = bencode.NewEncoder(f).Encode(mi)
if err != nil {
log.Errorf("error writing torrent metainfo file: %s", err)
return err
}
for k, m := range queue {
if m == magnet {
queue = append(queue[:k], queue[k+1:]...)
}
defer f.Close()
err = bencode.NewEncoder(f).Encode(mi)
if err != nil {
log.Errorf("error writing torrent metainfo file: %s", err)
return
}
for k, m := range queue {
if m == magnet {
queue = append(queue[:k], queue[k+1:]...)
}
}
log.Infof("New torrent file generated in: %s", file)
}()
}
log.Infof("New torrent file generated in: %s", file)
return nil
}
// GotFile will check if a torrent file exists and if not, try to generate it
func GotFile(torrent *models.Torrent) error {
var trackers []string
if torrent.Trackers == "" {
trackers = config.Get().Torrents.Trackers.Default
} else {
trackers = torrent.GetTrackersArray()
}
// We generate a new magnet link with all trackers (ours + ones from uploader)
magnet := format.InfoHashToMagnet(strings.TrimSpace(torrent.Hash), torrent.Name, trackers...)
//Check if file exists and open
_, err := os.Open(torrent.GetPath())
if err != nil {
err := GenerateTorrent(magnet)
if err != errPending && err != nil {
return err
}
if err == errPending {
for {
//Check again if file exists and open
_, err := os.Open(torrent.GetPath())
if err == nil {
break
}
// We check every 10 seconds
time.Sleep(10000)
}
}
}
return nil
}

367
utils/upload/multi.go Fichier normal
Voir le fichier

@ -0,0 +1,367 @@
package upload
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/NyaaPantsu/nyaa/config"
"github.com/NyaaPantsu/nyaa/models"
"github.com/NyaaPantsu/nyaa/utils/cache"
"github.com/NyaaPantsu/nyaa/utils/log"
)
const (
anidex = iota
nyaasi
ttosho
)
const (
pendingState = iota + 1
errorState
doneState
)
// Each service gives a status and a message when uploading to them
type service struct {
Status int `json:"status"`
Message string `json:"message"`
}
// MultipleForm is a struct used to follow the status of the uploads
type MultipleForm struct {
PantsuID uint `json:"id"`
Anidex service `json:"anidex"`
Nyaasi service `json:"nyaasi"`
TTosho service `json:"ttosho"`
}
// ToAnidex : function to upload a torrent to anidex
// TODO: subCat and lang should be taken from torrent model and not asked to be typed again
// so making the conversion here would be better
func ToAnidex(torrent *models.Torrent, apiKey string, subCat string, lang string) {
uploadMultiple := MultipleForm{PantsuID: torrent.ID, Anidex: service{Status: pendingState}}
uploadMultiple.save(anidex)
log.Info("Create anidex instance")
// If the torrent is posted as anonymous or apikey is not set, we set it with default value
if apiKey == "" || (torrent.Hidden && apiKey != "") {
apiKey = config.Get().Upload.DefaultAnidexToken
}
if apiKey == "" { // You need to check that apikey is not empty even after config. Since it is left empty in config by default and is required
log.Errorf("ApiKey is empty, we can't upload to anidex for torrent %d", torrent.ID)
uploadMultiple.updateAndSave(anidex, errorState, "No ApiKey providen (required)")
return
}
extraParams := map[string]string{
//Required
"api_key": apiKey,
"subcat_id": subCat,
"group_id": "0",
"lang_id": lang,
//Optional
"description": torrent.Description,
"torrent_name": torrent.Name,
}
if config.IsSukebei() {
extraParams["hentai"] = "1"
}
if torrent.IsRemake() {
extraParams["reencode"] = "1"
}
if torrent.IsAnon() {
extraParams["private"] = "1"
}
request, err := newfileUploadRequest("https://anidex.info/api/", extraParams, "file", torrent.GetPath())
if err != nil {
log.CheckError(err)
return
}
client := &http.Client{}
rsp, err := client.Do(request)
if err != nil {
log.CheckError(err)
return
}
log.Info("Launch anidex http request")
if err != nil {
uploadMultiple.updateAndSave(anidex, errorState, "Error during the HTTP POST request")
log.CheckErrorWithMessage(err, "Error in request: %s")
return
}
defer rsp.Body.Close()
bodyByte, err := ioutil.ReadAll(rsp.Body)
if err != nil {
uploadMultiple.updateAndSave(anidex, errorState, "Unknown error")
log.CheckErrorWithMessage(err, "Error in parsing request: %s")
return
}
if uploadMultiple.Anidex.Status == pendingState {
uploadMultiple.Anidex.Message = string(bodyByte)
if strings.Contains(uploadMultiple.Anidex.Message, "http://") {
uploadMultiple.Anidex.Status = doneState
} else {
uploadMultiple.Anidex.Status = errorState
}
uploadMultiple.save(anidex)
log.Info("Anidex request done")
fmt.Println(uploadMultiple)
}
}
// ToNyaasi : function to upload a torrent to anidex
func ToNyaasi(username string, password string, torrent *models.Torrent) {
uploadMultiple := MultipleForm{PantsuID: torrent.ID, Nyaasi: service{Status: pendingState}}
uploadMultiple.Nyaasi.Message = "Sorry u are not allowed"
uploadMultiple.save(nyaasi)
log.Info("Create NyaaSi instance")
// If the torrent is posted as anonymous or apikey is not set, we set it with default value
if username == "" || (torrent.Hidden && username != "") {
username = config.Get().Upload.DefaultNyaasiUsername
password = config.Get().Upload.DefaultNyaasiPassword
}
if username == "" || password == "" { // You need to check that username AND password are not empty even after config. Since they are left empty in config by default and are required
log.Errorf("Username or Password is empty, we can't upload to Nyaa.si for torrent %d", torrent.ID)
uploadMultiple.updateAndSave(nyaasi, errorState, "No valid account providen (required)")
return
}
params := map[string]interface{}{
"name": torrent.Name,
"category": Category(nyaasi, torrent),
"information": "",
"description": torrent.Description,
"anonymous": torrent.IsAnon(),
"hidden": false,
"remake": torrent.IsRemake(),
"trusted": torrent.IsTrusted(),
}
torrentData, _ := json.Marshal(params)
extraParams := map[string]string{
"torrent_data": string(torrentData),
}
request, err := newfileUploadRequest("https://nyaa.si/api/upload", extraParams, "torrent", torrent.GetPath())
if err != nil {
log.CheckError(err)
return
}
request.SetBasicAuth(username, password)
client := &http.Client{}
rsp, err := client.Do(request)
if err != nil {
log.CheckError(err)
return
}
log.Info("Launch Nyaa.Si http request")
if err != nil {
uploadMultiple.updateAndSave(nyaasi, errorState, "Error during the HTTP POST request")
log.CheckErrorWithMessage(err, "Error in request: %s")
return
}
defer rsp.Body.Close()
bodyByte, err := ioutil.ReadAll(rsp.Body)
if err != nil {
uploadMultiple.updateAndSave(nyaasi, errorState, "Unknown error")
log.CheckErrorWithMessage(err, "Error in parsing request: %s")
return
}
if uploadMultiple.Nyaasi.Status == pendingState {
var data map[string]interface{}
if err = json.Unmarshal(bodyByte, &data); err != nil {
log.CheckErrorWithMessage(err, "Cannot unmarshal json Response after upload request to Nyaa.Si")
uploadMultiple.Nyaasi.Status = errorState
uploadMultiple.Nyaasi.Message = err.Error()
}
if _, ok := data["errors"]; ok {
uploadMultiple.Nyaasi.Status = errorState
uploadMultiple.Nyaasi.Message = string(bodyByte)
} else {
uploadMultiple.Nyaasi.Status = doneState
uploadMultiple.Nyaasi.Message = fmt.Sprintf("%s", data["url"].(string))
}
uploadMultiple.save(nyaasi)
log.Info("Nyaa.Si request done")
fmt.Println(uploadMultiple)
}
}
// ToTTosho : function to upload a torrent to TokyoTosho
func ToTTosho(apiKey string, torrent *models.Torrent) {
uploadMultiple := MultipleForm{PantsuID: torrent.ID, TTosho: service{Status: pendingState}}
uploadMultiple.save(ttosho)
log.Info("Create TokyoTosho instance")
// If the torrent is posted as anonymous or apikey is not set, we set it with default value
if apiKey == "" || (torrent.Hidden && apiKey != "") {
apiKey = config.Get().Upload.DefaultTokyoTToken
}
if apiKey == "" { // You need to check that apikey is not empty even after config. Since it is left empty in config by default and is required
log.Errorf("ApiKey is empty, we can't upload to TokyoTosho for torrent %d", torrent.ID)
uploadMultiple.updateAndSave(ttosho, errorState, "No ApiKey providen (required)")
return
}
extraParams := map[string]string{
//Required
"apikey": apiKey,
"url": torrent.Download(),
"type": Category(ttosho, torrent),
"send": "true",
//Optional
"website": torrent.WebsiteLink,
"comment": torrent.Description,
}
request, err := newUploadRequest("https://www.tokyotosho.info/new.php", extraParams)
if err != nil {
log.CheckError(err)
return
}
client := &http.Client{}
rsp, err := client.Do(request)
if err != nil {
log.CheckError(err)
return
}
log.Info("Launch TokyoTosho http request")
if err != nil {
uploadMultiple.updateAndSave(ttosho, errorState, "Error during the HTTP POST request")
log.CheckErrorWithMessage(err, "Error in request: %s")
return
}
defer rsp.Body.Close()
bodyByte, err := ioutil.ReadAll(rsp.Body)
if err != nil {
uploadMultiple.updateAndSave(ttosho, errorState, "Unknown error")
log.CheckErrorWithMessage(err, "Error in parsing request: %s")
return
}
if uploadMultiple.TTosho.Status == pendingState {
if strings.Contains(string(bodyByte), "OK,") {
uploadMultiple.TTosho.Status = doneState
idnumber := strings.Split(string(bodyByte), ",")
uploadMultiple.TTosho.Message = fmt.Sprintf("Upload done! https://www.tokyotosho.info/details.php?id=%s", idnumber[1])
} else {
uploadMultiple.TTosho.Status = errorState
uploadMultiple.TTosho.Message = string(bodyByte)
}
uploadMultiple.save(ttosho)
log.Info("TokyoTosho request done")
fmt.Println(uploadMultiple)
}
}
// Saves the multipleform in each go routines and share the state of each upload for 5 minutes
// After timeout, the multipleform is flushed from memory
func (u *MultipleForm) save(which int) {
// We check if there is already a variable shared, if there is we update only the status/message of one service
if found, ok := cache.C.Get(fmt.Sprintf("tstatus_%d", u.PantsuID)); ok {
uploadStatus := found.(MultipleForm)
switch which {
case anidex:
uploadStatus.Anidex = u.Anidex
break
case nyaasi:
uploadStatus.Nyaasi = u.Nyaasi
break
case ttosho:
uploadStatus.TTosho = u.TTosho
break
}
u = &uploadStatus
}
// And then we save the variable in cache
cache.C.Set(fmt.Sprintf("tstatus_%d", u.PantsuID), *u, 5*time.Minute)
}
// shortcut function to update and save a service
func (u *MultipleForm) updateAndSave(which int, code int, message string) {
switch which {
case anidex:
u.Anidex.update(code, message)
break
case nyaasi:
u.Nyaasi.update(code, message)
break
case ttosho:
u.TTosho.update(code, message)
break
}
u.save(which)
}
// shortcut function to update both code and message of a service
func (s *service) update(code int, message string) {
s.Status = code
s.Message = message
}
// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
fi, err := file.Stat()
if err != nil {
return nil, err
}
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(paramName, fi.Name())
if err != nil {
return nil, err
}
if _, err = io.Copy(part, file); err != nil {
return nil, err
}
for key, val := range params {
_ = writer.WriteField(key, val)
}
err = writer.Close()
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", uri, body)
if err != nil {
return nil, err
}
request.Header.Add("Content-Type", writer.FormDataContentType())
return request, nil
}
// Creates a new upload http request with optional extra params
func newUploadRequest(uri string, params map[string]string) (*http.Request, error) {
var form url.Values
for key, val := range params {
form[key] = append(form[key], val)
}
body := bytes.NewBufferString(form.Encode())
request, err := http.NewRequest("POST", uri, body)
if err != nil {
return nil, err
}
return request, nil
}

Voir le fichier

@ -1,11 +1,6 @@
package upload
import (
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"os"
"reflect"
"strings"
@ -123,7 +118,7 @@ func ExtractInfo(c *gin.Context, r *torrentValidator.TorrentRequest) error {
return err
}
tfile, err := r.ValidateMultipartUpload(c, uploadFormTorrent)
err = r.ValidateMultipartUpload(c, uploadFormTorrent)
if err != nil {
return err
}
@ -133,16 +128,8 @@ func ExtractInfo(c *gin.Context, r *torrentValidator.TorrentRequest) error {
if err != nil {
return err
}
// after data has been checked & extracted, write it to disk
if len(config.Get().Torrents.FileStorage) > 0 && r.Filesize > 0 {
err := writeTorrentToDisk(tfile, r.Infohash+".torrent", &r.Filepath)
if err != nil {
return err
}
} else {
r.Filepath = ""
}
// We are not saving the file here because we need to add our own tracker list to the torrent file, therefore, we generate the torrent file at upload time through GenerateTorrent()
// when it is magnet
return nil
}
@ -213,19 +200,6 @@ func UpdateUnscopeTorrent(r *torrentValidator.UpdateRequest, t *models.Torrent,
return t
}
func writeTorrentToDisk(file multipart.File, name string, fullpath *string) error {
_, seekErr := file.Seek(0, io.SeekStart)
if seekErr != nil {
return seekErr
}
b, err := ioutil.ReadAll(file)
if err != nil {
return err
}
*fullpath = fmt.Sprintf("%s%c%s", config.Get().Torrents.FileStorage, os.PathSeparator, name)
return ioutil.WriteFile(*fullpath, b, 0644)
}
// NewTorrentRequest : creates a new torrent request struc with some default value
func NewTorrentRequest(params ...string) *torrentValidator.TorrentRequest {
torrentRequest := &torrentValidator.TorrentRequest{}

Voir le fichier

@ -15,3 +15,4 @@ var errTorrentHashInvalid = errors.New("torrent_hash_invalid")
var errTorrentTagsInvalid = errors.New("torrent_tags_invalid")
var errMissingFieldConfig = errors.New("Torrent Configuration in config.yml is invalid, missing a 'field' attribute in tags types")
var errWrongFieldConfig = errors.New("Torrent Configuration in config.yml is invalid, wrong 'field' attribute in tags types")
var errInvalidTorrentFile = errors.New("torrent_file_invalid")

Voir le fichier

@ -4,9 +4,8 @@ import (
"encoding/base32"
"encoding/hex"
"fmt"
"io"
"mime/multipart"
"net/url"
"os"
"reflect"
"regexp"
"strconv"
@ -15,13 +14,11 @@ import (
"github.com/NyaaPantsu/nyaa/config"
"github.com/NyaaPantsu/nyaa/utils/categories"
"github.com/NyaaPantsu/nyaa/utils/cookies"
"github.com/NyaaPantsu/nyaa/utils/format"
msg "github.com/NyaaPantsu/nyaa/utils/messages"
"github.com/NyaaPantsu/nyaa/utils/metainfo"
"github.com/NyaaPantsu/nyaa/utils/torrentLanguages"
"github.com/NyaaPantsu/nyaa/utils/validator/tag"
"github.com/anacrolix/torrent/metainfo"
"github.com/gin-gonic/gin"
"github.com/zeebo/bencode"
)
// ValidateName is a function validating the torrent name
@ -191,80 +188,90 @@ func (r *TorrentRequest) ExtractLanguage() error {
}
// ValidateMultipartUpload : Check if multipart upload is valid
func (r *TorrentRequest) ValidateMultipartUpload(c *gin.Context, uploadFormTorrent string) (multipart.File, error) {
func (r *TorrentRequest) ValidateMultipartUpload(c *gin.Context, uploadFormTorrent string) error {
// first: parse torrent file (if any) to fill missing information
tfile, _, err := c.Request.FormFile(uploadFormTorrent)
if err == nil {
var torrent metainfo.TorrentFile
// decode torrent
_, seekErr := tfile.Seek(0, io.SeekStart)
if seekErr != nil {
return tfile, seekErr
}
err = bencode.NewDecoder(tfile).Decode(&torrent)
torrent, err := metainfo.Load(tfile)
if err != nil {
return tfile, metainfo.ErrInvalidTorrentFile
return err
}
torrentInfos, err := torrent.UnmarshalInfo()
if err != nil {
return err
}
// check a few things
if torrent.IsPrivate() {
return tfile, errTorrentPrivate
if torrentInfos.Private != nil && *torrentInfos.Private {
return errTorrentPrivate
}
trackers := torrent.GetAllAnnounceURLS()
r.Trackers = CheckTrackers(trackers)
// We check that the trackers are valid,
// we filter the dead ones
// and we add the needed ones from our config in the request and the torrent file
r.Trackers = CheckTrackers(torrent)
if len(r.Trackers) == 0 {
return tfile, errTorrentNoTrackers
return errTorrentNoTrackers
}
// Name
// For the name, if none is provided in request, we set the name from the torrent file
if len(r.Name) == 0 {
r.Name = torrent.TorrentName()
r.Name = torrentInfos.Name
}
// Magnet link: if a file is provided it should be empty
if len(r.Magnet) != 0 {
return tfile, errTorrentAndMagnet
return errTorrentAndMagnet
}
_, seekErr = tfile.Seek(0, io.SeekStart)
if seekErr != nil {
return tfile, seekErr
}
infohash, err := metainfo.DecodeInfohash(tfile)
if err != nil {
return tfile, metainfo.ErrInvalidTorrentFile
}
r.Infohash = infohash
r.Magnet = format.InfoHashToMagnet(infohash, r.Name, trackers...)
// We are using here default functions from anacrolix
infohash := torrent.HashInfoBytes()
if infohash.String() == "" {
return errInvalidTorrentFile
}
r.Infohash = infohash.String()
r.Magnet = torrent.Magnet(r.Name, infohash).String()
// extract filesize
r.Filesize = int64(torrent.TotalSize())
r.Filesize = torrentInfos.TotalLength()
// extract filelist
fileInfos := torrent.Info.GetFiles()
fileInfos := torrentInfos.Files
for _, fileInfo := range fileInfos {
r.FileList = append(r.FileList, uploadedFile{
Path: fileInfo.Path,
Filesize: int64(fileInfo.Length),
Filesize: fileInfo.Length,
})
}
// Since we can change with anacrolix the trackers of a torrent,
// We will use this to generate directly the torrent file here
file := fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, infohash.String())
f, err := os.Create(file)
if err != nil {
return err
}
defer f.Close()
err = torrent.Write(f)
if err != nil {
return err
}
} else {
err = r.ValidateMagnet()
if err != nil {
return tfile, err
return err
}
err = r.ValidateHash()
if err != nil {
return tfile, err
return err
}
// TODO: Get Trackers from magnet URL
r.Filesize = 0
r.Filepath = ""
return tfile, nil
return nil
}
return tfile, err
return err
}
// ExtractInfo : Function to assign values from request to ReassignForm

Voir le fichier

@ -3,40 +3,61 @@ package torrentValidator
import (
"net/url"
"strings"
"github.com/NyaaPantsu/nyaa/config"
"github.com/anacrolix/torrent/metainfo"
)
// CheckTrackers : Check if there is good trackers in torrent
func CheckTrackers(trackers []string) []string {
func CheckTrackers(t *metainfo.MetaInfo) []string {
// TODO: move to runtime configuration
var deadTrackers = []string{ // substring matches!
"://open.nyaatorrents.info:6544",
"://tracker.openbittorrent.com:80",
"://tracker.publicbt.com:80",
"://stats.anisource.net:2710",
"://exodus.desync.com",
"://open.demonii.com:1337",
"://tracker.istole.it:80",
"://tracker.ccc.de:80",
"://bt2.careland.com.cn:6969",
"://announce.torrentsmd.com:8080",
"://open.demonii.com:1337",
"://tracker.btcake.com",
"://tracker.prq.to",
"://bt.rghost.net"}
var deadTrackers = config.Get().Torrents.Trackers.DeadTrackers
var trackerRet []string
for _, t := range trackers {
urlTracker, err := url.Parse(t)
if err == nil {
good := true
for _, check := range deadTrackers {
if strings.Contains(t, check) {
good = false
break // No need to continue the for loop
trackerRet := []string{}
tempList := metainfo.AnnounceList{}
for _, group := range t.AnnounceList {
var trackers []string
for _, tracker := range group {
urlTracker, err := url.ParseRequestURI(tracker)
if err == nil {
good := true
for _, check := range deadTrackers {
// the tracker is part of the deadtracker list
// we don't keep it
if strings.Contains(tracker, check) {
good = false
break // No need to continue the for loop
}
}
if good {
// We only keep the good trackers
trackers = append(trackers, urlTracker.String())
}
}
if good {
trackerRet = append(trackerRet, urlTracker.String())
}
if len(trackers) > 0 {
tempList = append(tempList, trackers)
trackerRet = append(trackerRet, trackers...)
}
}
t.AnnounceList = tempList
defaultTracker := config.Get().Torrents.Trackers.GetDefault()
if defaultTracker != "" {
t.Announce = defaultTracker
}
for _, key := range config.Get().Torrents.Trackers.NeededTrackers {
inside := false
if key < len(config.Get().Torrents.Trackers.Default) {
tracker := config.Get().Torrents.Trackers.Default[key]
for _, tr := range trackerRet {
if tr == tracker {
inside = true
}
}
if !inside {
trackerRet = append(trackerRet, tracker)
t.AnnounceList = append(t.AnnounceList, []string{tracker})
}
}
}

Voir le fichier

@ -0,0 +1,46 @@
package torrentValidator
import (
"testing"
"github.com/anacrolix/torrent/metainfo"
"github.com/stretchr/testify/assert"
)
func TestCheckTrackers(t *testing.T) {
assert := assert.New(t)
tests := []struct {
Info metainfo.MetaInfo
Expected []string
ExpectedList metainfo.AnnounceList
}{
{
metainfo.MetaInfo{
AnnounceList: [][]string{{"udp://nn.fof:4545/ano", "lol"}, {"://nyaa.tr/ann"}, {".co", "http://mont.co:444/ann"}},
},
[]string{"udp://nn.fof:4545/ano", "http://mont.co:444/ann", "udp://tracker.uw0.xyz:6969/announce", "http://anidex.moe:6969/announce"},
[][]string{{"udp://nn.fof:4545/ano"}, {"http://mont.co:444/ann"}, {"udp://tracker.uw0.xyz:6969/announce"}, {"http://anidex.moe:6969/announce"}},
},
{
metainfo.MetaInfo{
AnnounceList: [][]string{{"http://open.nyaatorrents.info:6544", "http://mont.co:444/ann"}}, // dead tracker
},
[]string{"http://mont.co:444/ann", "udp://tracker.uw0.xyz:6969/announce", "http://anidex.moe:6969/announce"},
[][]string{{"http://mont.co:444/ann"}, {"udp://tracker.uw0.xyz:6969/announce"}, {"http://anidex.moe:6969/announce"}},
},
{
metainfo.MetaInfo{
AnnounceList: [][]string{{"http://open.nyaatorrents.info:6544", "http://mont.co:444/ann", "http://mont.co:4434/ann"}, {"http://mont.co:444/anno"}}, // dead tracker
},
[]string{"http://mont.co:444/ann", "http://mont.co:4434/ann", "http://mont.co:444/anno", "udp://tracker.uw0.xyz:6969/announce", "http://anidex.moe:6969/announce"},
[][]string{{"http://mont.co:444/ann", "http://mont.co:4434/ann"}, {"http://mont.co:444/anno"}, {"udp://tracker.uw0.xyz:6969/announce"}, {"http://anidex.moe:6969/announce"}},
},
}
for _, test := range tests {
trackers := CheckTrackers(&test.Info)
assert.Equal(test.Expected, trackers)
assert.Equal(test.ExpectedList, test.Info.AnnounceList)
}
}