Merge branch 'dev' into filelist-fetching

config/default_config.yml

@@ -54,6 +54,23 @@ scraper:
   max_stat_scraping_frequency: 5
 # MaxStatScrapingFrequencyUnknown : On-demand stat scraping for unknown torrents can only be called every X minutes
   max_stat_scraping_frequency_unknown: 2
+#  Default config for upload
+upload:
+  # The following token/accounts are used when an user wants to upload to another website
+  # but doesn't have his own API token.
+  anidex_api_token:
+  # TokyoTosho : Be aware that file upload doesn't work with the host (only url upload)
+  # so you need to enable users to download without ddos protection
+  tokyot_api_token:
+  # Nyaa.Si : we need a username and a password for the account upload, please use a difficult one! 
+  nyaasi_username:
+  nyaasi_password:
+# UploadsDisabled : Disable uploads for everyone except below
+  uploads_disabled: false
+# AdminsAreStillAllowedTo : Enable admin torrent upload even if UploadsDisabled is true
+  admins_are_still_allowed_to: true
+# TrustedUsersAreStillAllowedTo : Enable trusted users torrent upload even if UploadsDisabled is true
+  trusted_users_are_still_allowed_to: true
 # Config by default for the cache
 cache:
   dialect: nop
@@ -115,12 +132,6 @@ torrents:
   storage_link: /download/%s
 # TorrentCacheLink : Url of torrent site cache
   cache_link: #http://anicache.com/torrent/%s.torrent
-# UploadsDisabled : Disable uploads for everyone except below
-  uploads_disabled: false
-# AdminsAreStillAllowedTo : Enable admin torrent upload even if UploadsDisabled is true
-  admins_are_still_allowed_to: true
-# TrustedUsersAreStillAllowedTo : Enable trusted users torrent upload even if UploadsDisabled is true
-  trusted_users_are_still_allowed_to: true
   trackers:
 # Trackers : Default trackers supported
     default:
@@ -140,6 +151,22 @@ torrents:
 # NeededTrackers : Array indexes of Trackers for needed tracker in a torrent file
     needed:
       - 0
+      - 12
+    dead:
+      - ://open.nyaatorrents.info:6544
+      - ://tracker.openbittorrent.com:80
+      - ://tracker.publicbt.com:80
+      - ://stats.anisource.net:2710
+      - ://exodus.desync.com
+      - ://open.demonii.com:1337
+      - ://tracker.istole.it:80
+      - ://tracker.ccc.de:80
+      - ://bt2.careland.com.cn:6969
+      - ://announce.torrentsmd.com:8080
+      - ://open.demonii.com:1337
+      - ://tracker.btcake.com
+      - ://tracker.prq.to
+      - ://bt.rghost.net
 # TorrentOrder : Default sorting field for torrents
   order: date
 # TorrentSort : Default sorting order for torrents

config/structs.go

@@ -107,27 +107,28 @@ type I18nConfig struct {
 
 // ScrapeConfig : Config struct for Scraping
 type ScrapeConfig struct {
-	URL                              string           `json:"scrape_url" yaml:"url,omitempty"`
+	URL             string `json:"scrape_url" yaml:"url,omitempty"`
-	Name                             string           `json:"name"  yaml:"name,omitempty"`
+	Name            string `json:"name"  yaml:"name,omitempty"`
-	IntervalSeconds                  int64            `json:"interval" yaml:"interval,omitempty"`
+	IntervalSeconds int64  `json:"interval" yaml:"interval,omitempty"`
 }
 
 // ScraperConfig :  Config struct for Scraper
 type ScraperConfig struct {
-	Addr                             string           `json:"bind" yaml:"addr,omitempty"`
+	Addr                            string         `json:"bind" yaml:"addr,omitempty"`
-	NumWorkers                       int              `json:"workers" yaml:"workers,omitempty"`
+	NumWorkers                      int            `json:"workers" yaml:"workers,omitempty"`
-	IntervalSeconds                  int64            `json:"default_interval" yaml:"default_interval,omitempty"`
+	IntervalSeconds                 int64          `json:"default_interval" yaml:"default_interval,omitempty"`
-	Trackers                         []ScrapeConfig   `json:"trackers" yaml:"trackers,omitempty"`
+	Trackers                        []ScrapeConfig `json:"trackers" yaml:"trackers,omitempty"`
-	StatScrapingFrequency            float64          `json:"stat_scraping_frequency" yaml:"stat_scraping_frequency,omitempty"`
+	StatScrapingFrequency           float64        `json:"stat_scraping_frequency" yaml:"stat_scraping_frequency,omitempty"`
-	StatScrapingFrequencyUnknown     float64          `json:"stat_scraping_frequency_unknown" yaml:"stat_scraping_frequency_unknown,omitempty"`
+	StatScrapingFrequencyUnknown    float64        `json:"stat_scraping_frequency_unknown" yaml:"stat_scraping_frequency_unknown,omitempty"`
-	MaxStatScrapingFrequency         float64          `json:"max_stat_scraping_frequency" yaml:"max_stat_scraping_frequency,omitempty"`
+	MaxStatScrapingFrequency        float64        `json:"max_stat_scraping_frequency" yaml:"max_stat_scraping_frequency,omitempty"`
-	MaxStatScrapingFrequencyUnknown  float64          `json:"max_stat_scraping_frequency_unknown" yaml:"max_stat_scraping_frequency_unknown,omitempty"`
+	MaxStatScrapingFrequencyUnknown float64        `json:"max_stat_scraping_frequency_unknown" yaml:"max_stat_scraping_frequency_unknown,omitempty"`
 }
 
 // TrackersConfig ; Config struct for Trackers
 type TrackersConfig struct {
 	Default        ArrayString `yaml:"default,flow,omitempty"`
 	NeededTrackers []int       `yaml:"needed,flow,omitempty"`
+	DeadTrackers   ArrayString `yaml:"dead,flow,omitempty"`
 }
 
 // TorrentsConfig : Config struct for Torrents
@@ -151,12 +152,13 @@ type TorrentsConfig struct {
 
 // UploadConfig : Config struct for uploading torrents
 type UploadConfig struct {
-	DefaultAnidexToken            string            `yaml:"anidex_api_token,omitempty"`
+	DefaultAnidexToken            string `yaml:"anidex_api_token,omitempty"`
-	DefaultNyaasiToken            string            `yaml:"nyaasi_api_token,omitempty"`
+	DefaultNyaasiUsername         string `yaml:"nyaasi_api_username,omitempty"`
-	DefaultTokyoTToken            string            `yaml:"tokyot_api_token,omitempty"`
+	DefaultNyaasiPassword         string `yaml:"nyaasi_api_password,omitempty"`
-	UploadsDisabled               bool              `yaml:"uploads_disabled,omitempty"`
+	DefaultTokyoTToken            string `yaml:"tokyot_api_token,omitempty"`
-	AdminsAreStillAllowedTo       bool              `yaml:"admins_are_still_allowed_to,omitempty"`
+	UploadsDisabled               bool   `yaml:"uploads_disabled,omitempty"`
-	TrustedUsersAreStillAllowedTo bool              `yaml:"trusted_users_are_still_allowed_to,omitempty"`
+	AdminsAreStillAllowedTo       bool   `yaml:"admins_are_still_allowed_to,omitempty"`
+	TrustedUsersAreStillAllowedTo bool   `yaml:"trusted_users_are_still_allowed_to,omitempty"`
 }
 
 // UsersConfig : Config struct for Users
@@ -225,9 +227,9 @@ type ModelsConfig struct {
 }
 
 type DefaultThemeConfig struct {
-	Theme    string `yaml:"theme,omitempty"`
+	Theme  string `yaml:"theme,omitempty"`
-	Dark     string `yaml:"dark,omitempty"`
+	Dark   string `yaml:"dark,omitempty"`
-	Forced   string `yaml:"forced,omitempty"`
+	Forced string `yaml:"forced,omitempty"`
 }
 
 // SearchConfig : Config struct for search
@@ -275,3 +277,14 @@ func (ty TagTypes) Get(tagType string) TagType {
 	}
 	return TagType{}
 }
+
+// GetDefault returns the first tracker from the needed ones
+func (tc TrackersConfig) GetDefault() string {
+	if len(tc.NeededTrackers) > 0 {
+		return tc.Default[tc.NeededTrackers[0]]
+	}
+	if len(tc.Default) > 0 {
+		return tc.Default[0]
+	}
+	return ""
+}

controllers/torrent/download.go

@@ -31,16 +31,16 @@ func DownloadTorrent(c *gin.Context) {
 		templates.Render(c, "errors/torrent_file_missing.jet.html", variables)
 		return
 	}
-	
+
 	if c.Query("js_query") != "" {
 		exists := true
 		generating := false
-		
+
 		if len(config.Get().Torrents.FileStorage) == 0 {
 			exists = false
 		} else {
 			Openfile, err := os.Open(fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, hash))
-			defer Openfile.Close() 
+			defer Openfile.Close()
 			if err != nil {
 				exists = false
 				generating = true
@@ -52,14 +52,11 @@ func DownloadTorrent(c *gin.Context) {
 					trackers = torrent.GetTrackersArray()
 				}
 				magnet := format.InfoHashToMagnet(strings.TrimSpace(torrent.Hash), torrent.Name, trackers...)
-				if upload.GenerateTorrent(magnet) != nil {
+				go upload.GenerateTorrent(magnet)
-					//Error during the generation
-					generating = false
-				}
 			}
 		}
 		c.JSON(200, gin.H{ // Better to use gin for that, less code
-			"exists": exists,
+			"exists":     exists,
 			"generating": generating,
 		})
 		return
@@ -82,7 +79,7 @@ func DownloadTorrent(c *gin.Context) {
 	}
 
 	//Check if file exists and open
-	Openfile, err := os.Open(fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, hash))
+	Openfile, err := os.Open(torrent.GetPath())
 	if err != nil {
 		//File not found, send 404
 		variables := templates.Commonvariables(c)
@@ -94,9 +91,7 @@ func DownloadTorrent(c *gin.Context) {
 		}
 		magnet := format.InfoHashToMagnet(strings.TrimSpace(torrent.Hash), torrent.Name, trackers...)
 		variables.Set("magnet", magnet)
-		if upload.GenerateTorrent(magnet) != nil {
+		go upload.GenerateTorrent(magnet)
-			messages.AddError("errors", "Could not generate torrent file")	
-		}
 		templates.Render(c, "errors/torrent_file_missing.jet.html", variables)
 		return
 	}

controllers/upload/router.go

@@ -4,4 +4,5 @@ import "github.com/NyaaPantsu/nyaa/controllers/router"
 
 func init() {
 	router.Get().Any("/upload", UploadHandler)
+	router.Get().Any("/upload/status/:id", multiUploadStatus)
 }

controllers/upload/upload.go

@@ -2,6 +2,7 @@ package uploadController
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"strconv"
 
@@ -9,13 +10,14 @@ import (
 	"github.com/NyaaPantsu/nyaa/models"
 	"github.com/NyaaPantsu/nyaa/models/torrents"
 	"github.com/NyaaPantsu/nyaa/templates"
+	"github.com/NyaaPantsu/nyaa/utils/cache"
 	"github.com/NyaaPantsu/nyaa/utils/captcha"
+	"github.com/NyaaPantsu/nyaa/utils/log"
 	msg "github.com/NyaaPantsu/nyaa/utils/messages"
 	"github.com/NyaaPantsu/nyaa/utils/publicSettings"
 	"github.com/NyaaPantsu/nyaa/utils/upload"
 	"github.com/NyaaPantsu/nyaa/utils/validator/torrent"
 	"github.com/gin-gonic/gin"
-	"github.com/NyaaPantsu/nyaa/utils/log"
 )
 
 // UploadHandler : Main Controller for uploading a torrent
@@ -67,10 +69,53 @@ func UploadPostHandler(c *gin.Context) {
 		messages.AddError("errors", err.Error())
 	}
 
+	AnidexUpload := false
+	NyaaSiUpload := false
+	TokyoToshoUpload := false
+
+	if c.PostForm("anidex_api") != "" || c.PostForm("anidex_upload") == "true" {
+		AnidexUpload = true
+	}
+	if c.PostForm("nyaasi_api") != "" || c.PostForm("nyaasi_upload") == "true" {
+		NyaaSiUpload = true
+	}
+	if c.PostForm("tokyot_api") != "" || c.PostForm("tokyot_upload") == "true" {
+		TokyoToshoUpload = true
+	}
+
 	if !messages.HasErrors() {
-		// add to db and redirect
+		// add to db
 		torrent, err := torrents.Create(user, &uploadForm)
 		log.CheckErrorWithMessage(err, "ERROR_TORRENT_CREATED: Error while creating entry in db")
+
+		if AnidexUpload || NyaaSiUpload || TokyoToshoUpload {
+			go func(anidexApiKey string, anidexFormCategory string, anidexFormLang string, nyaasiUsername string, nyaasiPassword string, toshoApiKey string) {
+				err := upload.GotFile(torrent)
+				if err != nil {
+					log.CheckError(err)
+					return
+				}
+				// User wants to upload to other websites too
+				if AnidexUpload {
+					go upload.ToAnidex(torrent, anidexApiKey, anidexFormCategory, anidexFormLang)
+				}
+
+				if NyaaSiUpload {
+					go upload.ToNyaasi(nyaasiUsername, nyaasiPassword, torrent)
+				}
+
+				if TokyoToshoUpload {
+					go upload.ToTTosho(toshoApiKey, torrent)
+				}
+			}(c.PostForm("anidex_api"), c.PostForm("anidex_form_category"), c.PostForm("anidex_form_lang"), c.PostForm("nyaasi_username"), c.PostForm("nyaasi_password"), c.PostForm("tokyot_api"))
+			// After that, we redirect to the page for upload status
+			url := fmt.Sprintf("/upload/status/%d", torrent.ID)
+			c.Redirect(302, url)
+			return
+		}
+
+		// We don't need it to be synchronous since the generation can be left in a background process
+		go upload.GotFile(torrent)
 		url := "/view/" + strconv.FormatUint(uint64(torrent.ID), 10)
 		c.Redirect(302, url+"?success")
 	}
@@ -88,3 +133,27 @@ func UploadGetHandler(c *gin.Context) {
 	}
 	templates.Form(c, "site/torrents/upload.jet.html", uploadForm)
 }
+
+// multiUploadStatus : controller to show the multi upload status
+func multiUploadStatus(c *gin.Context) {
+	id := c.Param("id")
+	if id == "" {
+		c.AbortWithStatus(http.StatusNotFound)
+		return
+	}
+	if found, ok := cache.C.Get("tstatus_" + id); ok {
+		uploadMultiple := found.(upload.MultipleForm)
+		// if ?json we print the json format
+		if _, ok = c.GetQuery("json"); ok {
+			c.JSON(http.StatusOK, uploadMultiple)
+			return
+		}
+		// else we send the upload multiple form (support of manual F5)
+		variables := templates.Commonvariables(c)
+		variables.Set("UploadMultiple", uploadMultiple)
+		templates.Render(c, "site/torrents/upload_multiple.jet.html", variables)
+	} else {
+		// here it means the upload status is already flushed from memory
+		c.AbortWithStatus(http.StatusNotFound)
+	}
+}

models/torrent.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"html/template"
+	"os"
 	"path/filepath"
 	"reflect"
 	"strconv"
@@ -166,11 +167,22 @@ func (t *Torrent) IsDeleted() bool {
 	return t.DeletedAt != nil
 }
 
+// IsAnon : Return if a torrent is displayed as anon
+// Be aware, it doesn't mean that the owner is anonymous!
+func (t *Torrent) IsAnon() bool {
+	return t.Hidden || t.UploaderID == 0
+}
+
 // GetDescriptiveTags : Return the descriptive tags
 func (t *Torrent) GetDescriptiveTags() string {
 	return t.AcceptedTags
 }
 
+// GetPath : Helpers to get the path to the torrent file
+func (t *Torrent) GetPath() string {
+	return fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, t.Hash)
+}
+
 // AddToESIndex : Adds a torrent to Elastic Search
 func (t Torrent) AddToESIndex(client *elastic.Client) error {
 	ctx := context.Background()
@@ -224,7 +236,7 @@ func (t *Torrent) ParseTrackers(trackers []string) {
 		}
 	}
 	trackers = tempTrackers
-		
+
 	v["tr"] = trackers
 	t.Trackers = v.Encode()
 }
@@ -349,23 +361,14 @@ func (t *Torrent) ToJSON() TorrentJSON {
 	} else if t.OldUploader != "" {
 		uploader = t.OldUploader
 	}
-	torrentlink := ""
+
-	if len(config.Get().Torrents.CacheLink) > 0 { // Only use torrent cache if set, don't check id since better to have all .torrent
-		if config.IsSukebei() {
-			torrentlink = "" // torrent cache doesn't have sukebei torrents
-		} else {
-			torrentlink = fmt.Sprintf(config.Get().Torrents.CacheLink, t.Hash)
-		}
-	} else if len(config.Get().Torrents.StorageLink) > 0 { // Only use own .torrent if storage set
-		torrentlink = fmt.Sprintf(config.Get().Torrents.StorageLink, t.Hash)
-	}
 	scrape := Scrape{}
 	if t.Scrape != nil {
 		scrape = *t.Scrape
 	}
-	
+
 	statsObsolete := []bool{false, false}
-	
+
 	if scrape.LastScrape.IsZero() || (scrape.Seeders == 0 && scrape.Leechers == 0 && scrape.Completed == 0) {
 		statsObsolete[0] = true
 		//The displayed stats are obsolete, S/D/L will show "Unknown"
@@ -374,7 +377,7 @@ func (t *Torrent) ToJSON() TorrentJSON {
 		statsObsolete[1] = true
 		//The stats need to be refreshed, either because they are valid and older than one month (not that reliable) OR if they are unknown but have been scraped 1h (or more) ago
 	}
-	
+
 	t.ParseLanguages()
 	res := TorrentJSON{
 		ID:            t.ID,
@@ -394,7 +397,7 @@ func (t *Torrent) ToJSON() TorrentJSON {
 		WebsiteLink:   sanitize.Safe(t.WebsiteLink),
 		Languages:     t.Languages,
 		Magnet:        template.URL(magnet),
-		TorrentLink:   sanitize.Safe(torrentlink),
+		TorrentLink:   sanitize.Safe(t.Download()),
 		Leechers:      scrape.Leechers,
 		Seeders:       scrape.Seeders,
 		Completed:     scrape.Completed,
@@ -541,11 +544,25 @@ func (t *Torrent) DeleteTags() {
 	}
 }
 
-func contains(s []string, e string) bool {
+// Download generate a download link for a torrent
-    for _, a := range s {
+func (t *Torrent) Download() (torrentlink string) {
-        if a == e {
+	if len(config.Get().Torrents.CacheLink) > 0 { // Only use torrent cache if set, don't check id since better to have all .torrent
-            return true
+		if !config.IsSukebei() { // torrent cache doesn't have sukebei torrents
-        }
+			torrentlink = fmt.Sprintf(config.Get().Torrents.CacheLink, t.Hash)
-    }
+		}
-    return false
+		return
+	}
+	if len(config.Get().Torrents.StorageLink) > 0 { // Only use own .torrent if storage set
+		torrentlink = fmt.Sprintf(config.Get().Torrents.StorageLink, t.Hash)
+	}
+	return
+}
+
+func contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
 }

public/css/themes/classic.css

@@ -70,6 +70,7 @@ a:hover, .h-nav a:hover, .h-nav:active a:hover  {
 .upload-form-table .checkbox-container+input {
     width: 385px;1
 }
+
 .upload-form-table .table-checkboxes {
     padding: 3px 0!important;
     width: 100%!important;
@@ -677,7 +678,7 @@ span.tag {
 }
 
 .upload-form-table {
-    width: auto;
+    width: 542px;
 }
 .upload-form-table .table-input-label label::after {
     content: ':';

public/css/themes/tomorrow.css

@@ -334,7 +334,11 @@ span.tag {
 
 .upload-form-table .checkbox-container {
     border-color: #0c0d0e;
-    background: #333438;
+    background: #28282b;
+}
+
+#anidex-upload-info > div input[type="text"], #anidex-upload-info > div select {
+    background-color: #28282b;
 }
 
 .torrent-info-row .tr-se span, .torrent-info-row .tr-le span, .torrent-info-row .tr-dl span {

public/js/main.js

@@ -59,8 +59,10 @@ function parseAllDates() {
   var list = document.getElementsByClassName("date-short")
   for(var i = 0; i < list.length; i++) {
     var e = list[i]
-    e.innerText = new Date(e.title).toLocaleString(lang, ymdOpt)
+    var newDate = new Date(e.title)
-    e.title = new Date(e.title).toLocaleString(lang)
+    if(isNaN(newDate.valueOf())) continue
+    e.innerText = newDate.toLocaleString(lang, ymdOpt)
+    e.title = newDate.toLocaleString(lang)
   }
 
   var list = document.getElementsByClassName("date-full")

templates/admin/guidelines.jet.html

@@ -1,8 +1,8 @@
-{{ extends "layouts/index_admin" }}
+{{ extends "layouts/index_admin" }}
-{{block title()}}{{ T("moderation_guidelines") }}{{end}}
+{{block title()}}{{ T("moderation_guidelines") }}{{end}}
-{{ block content_body()}}
+{{ block content_body()}}
-<div class="results box">
+<div class="results box">
-  <h1>{{ T("moderation_guidelines") }}</h1>
+  <h1>{{ T("moderation_guidelines") }}</h1>
-  <img src="https://www.themarysue.com/wp-content/uploads/2016/09/harassment-guide.png">
+  <img src="https://www.themarysue.com/wp-content/uploads/2016/09/harassment-guide.png">
-</div>
+</div>
-{{end}}
+{{end}}

templates/site/torrents/upload.jet.html

@@ -66,6 +66,90 @@
 		     <td class="table-input-label"><label for="desc">{{ T("torrent_description")}}</label></td>
 		     <td class="table-input markdown-container"><textarea name="desc" id="desc" class="form-input up-input" style="height: 10rem;">{{Form.Description}}</textarea></td>
 		  </tr>
+		  <tr>
+			<td class="table-input-label">{{ T("upload_to") }} Anidex, Nyaa.si, TokyoTosho:</td>
+			<td class="table-input">
+				 <details id="anidex-upload-info">
+				  <summary class="form-input">{{ T("upload_to_other") }}</summary>
+				  <div class="form-input">
+					<p>{{ T("upload_to") }} <b>Anidex</b>:</p>
+					<div>
+						<div class="checkbox-container"><input type="checkbox" value="true" name="anidex_upload" id="anidex_upload" class="form-torrent" {{if User.AnidexAPIToken != ""}}checked{{end}} /></div>
+						<input name="anidex_api" id="anidex_api" placeholder="Anidex API token" class="form-input" type="text" value="{{User.AnidexAPIToken}}" {{if User.AnidexAPIToken != ""}}disabled{{end}}/>
+						<br/>
+						<select class="form-input" name="anidex_form_category">
+							<option>{{ T("choose_category") }}</option>
+							<option value="1" style="background-color: #9f9;">Anime - Sub</option>
+							<option value="2" style="background-color: #9c9;">Anime - Raw</option>
+							<option value="3" style="background-color: #3f3;">Anime - Dub</option>
+							<option value="4" style="background-color: #9ff;">LA - Sub</option>
+							<option value="5" style="background-color: #9cc;">LA - Raw</option>
+							<option value="6" style="background-color: #ff6;">Light Novel</option>
+							<option value="7" style="background-color: #f9f;">Manga - Translated</option>
+							<option value="8" style="background-color: #c9c;">Manga - Raw</option>
+							<option value="9" style="background-color: #f99;">Audio - Lossy</option>
+							<option value="10" style="background-color: #c99;">Audio - Lossless</option>
+							<option value="11" style="background-color: #f66;">Video</option>
+							<option value="12" style="background-color: #ff9;">Games</option>
+							<option value="13" style="background-color: #cc9;">Applications</option>
+							<option value="14" style="background-color: #99c;">Pictures</option>
+							{{if Sukebei()}}<option value="15" style="background-color: #99f;">Adult Video</option>{{end}}
+							<option value="16" style="background-color: #999;">Other</option>
+						</select>
+						<select class="form-input" name="anidex_form_lang">
+							<option>{{ T("choose_language") }}</option>
+							<option value="1">English</option>
+							<option value="2">Japanese</option>
+							<option value="3">Polish</option>
+							<option value="4">Serbo-Croatian</option>
+							<option value="5">Dutch</option>
+							<option value="6">Italian</option>
+							<option value="7">Russian</option>
+							<option value="8">German</option>
+							<option value="9">Hungarian</option>
+							<option value="10">French</option>
+							<option value="11">Finnish</option>
+							<option value="12">Vietnamese</option>
+							<option value="13">Greek</option>
+							<option value="14">Bulgarian</option>
+							<option value="15">Spanish</option>
+							<option value="16">Portuguese (Brasil)</option>
+							<option value="17">Portuguese (Portugal)</option>
+							<option value="18">Swedish</option>
+							<option value="19">Arabic</option>
+							<option value="20">Danish</option>
+							<option value="21">Chinese (Simplified)</option>
+							<option value="22">Bengali</option>
+							<option value="23">Romanian</option>
+							<option value="24">Czech</option>
+							<option value="25">Mongolian</option>
+							<option value="26">Turkish</option>
+							<option value="27">Indonesian</option>
+							<option value="28">Korean</option>
+							<option value="29">Spanish (LATAM)</option>
+							<option value="30">Persian</option>
+							<option value="31">Malaysian</option>
+						</select>
+					</div>
+					
+					<p>{{ T("upload_to") }} <b>Nyaa.si</b>:</p>
+					<div>
+					<div class="checkbox-container"><input type="checkbox" value="true" name="nyaasi_upload" id="nyaasi_upload" class="form-torrent"/></div>
+					<input name="nyaasi_username" id="nyaasi_username" placeholder="Nyaa.si Username" class="form-input up-input" type="text" value="" />
+					<br/>
+					<input name="nyaasi_password" id="nyaasi_password" placeholder="Nyaa.si Password" class="form-input up-input" type="password" value="" />
+					</div>
+					
+					<p>{{ T("upload_to") }} <b>TokyoTosho</b>:</p>
+					<div>
+						<div class="checkbox-container"><input type="checkbox" value="true" name="tokyot_upload" id="tokyot_upload" {{if User.TokyoTAPIToken  != ""}}checked{{end}} class="form-torrent"/></div>
+						<input name="tokyot_api" id="tokyot_api" placeholder="TokyoTosho API token" class="form-input up-input" type="text" value="{{User.TokyoTAPIToken }}" {{if User.AnidexAPIToken != ""}}disabled{{end}}/>
+					</div>
+					
+				  </div>
+				</details>
+			</td>
+		  </tr>
 		  <tr>
 		     <td class="table-input-label">{{ T("torrent_tags")}}:</td>
 		     <td class="table-input">

templates/site/torrents/upload_multiple.jet.html

@@ -2,20 +2,30 @@
 {{block title()}}{{ T("upload")}}{{end}}
 {{block content_body()}}
 <div style="text-align: left;" class="box">
-	<h1 style="text-align:center;margin-top; 4px;">Upload status</h1>
+	<h1 style="text-align:center;margin-top: 4px;">{{ T("upload_status") }}</h1>
 	<table class="multiple-upload">
 		<tbody>
-			<tr><td colspan="4"><h3>Nyaa Pantsu upload status</h3></td></tr>
+			<tr><td colspan="4"><h3>Nyaa Pantsu {{ T("upload_status") }}</h3></td></tr>
-			<tr class="upload-status"><td class="upload-site-name">Nyaa Pantsu</td><td>Status:</td><td class="icon-finished finished"></td><td class="upload-progress finished">Finished</td></tr>
+			<tr class="upload-status"><td class="upload-site-name">Nyaa Pantsu</td><td>{{ T("status") }}:</td><td class="icon-finished finished"></td><td class="upload-progress finished">{{ T("finished") }}</td></tr>
-			<tr><td colspan="4" class="uploaded-url"><input type="text" class="form-input" placeholder="http://" value="http://nyaa.pantsu.cat/view/4556456" disabled></td></tr>
+			<tr><td colspan="4" class="uploaded-url"><a href="/view/{{UploadMultiple.PantsuID}}" target="_blank"><input type="text" class="form-input" value="{{if Sukebei()}}{{Config.WebAddress.Sukebei}}{{else}}{{Config.WebAddress.Nyaa}}{{end}}/view/{{UploadMultiple.PantsuID}}" disabled></a></td></tr>
 			
-			<tr><td colspan="4"><h3>Nyaa.si upload status</h3></td></tr>
+			{{ if UploadMultiple.Nyaasi.Status != 0 }}
-			<tr class="upload-status"><td class="upload-site-name">Nyaa.si</td><td>Status:</td><td class="icon-pending pending"></td><td class="upload-progress pending">Pending</td></tr>
+			<tr><td colspan="4"><h3>Nyaa.si {{ T("upload_status") }}</h3></td></tr>
-			<tr><td colspan="4" class="uploaded-url"><input type="text" class="form-input" placeholder="Uploading..." disabled></td></tr>
+			<tr class="upload-status nyaasi"><td class="upload-site-name">Nyaa.si</td><td>{{ T("status") }}:</td><td class="statusicon icon-{{ if UploadMultiple.Nyaasi.Status == 1}}pending pending{{else if UploadMultiple.Nyaasi.Status == 2}}error error{{else}}finished finished{{end}}"></td><td class="upload-progress {{ if UploadMultiple.Nyaasi.Status == 1}}pending{{else if UploadMultiple.Nyaasi.Status == 2}}error{{else}}finished{{end}}">{{ if UploadMultiple.Nyaasi.Status == 1}}{{ T("pending") }}{{else if UploadMultiple.Nyaasi.Status == 2}}{{ T("error") }}{{else}}{{ T("finished") }}{{end}}</td></tr>
+			<tr><td colspan="4" class="uploaded-url"><input type="text"id="nyaasiMess"  class="form-input" value="{{UploadMultiple.Nyaasi.Message}}" disabled></td></tr>
+			{{end}}
 			
-			<tr><td colspan="4"><h3>Anidex upload status</h3></td></tr>
+			{{ if UploadMultiple.Anidex.Status != 0 }}
-			<tr class="upload-status"><td class="upload-site-name">Anidex</td><td>Status:</td><td class="icon-error error"></td><td class="upload-progress error">Error</td></tr>
+			<tr><td colspan="4"><h3>Anidex {{ T("upload_status") }}</h3></td></tr>
-			<tr><td colspan="4" class="uploaded-url"><input type="text" class="form-input" placeholder="http://" value="Error: XXXX" disabled></td></tr>
+			<tr class="upload-status anidex"><td class="upload-site-name">Anidex</td><td>{{ T("status") }}:</td><td class="statusicon icon-{{ if UploadMultiple.Anidex.Status == 1}}pending pending{{else if UploadMultiple.Anidex.Status == 2}}error error{{else}}finished finished{{end}}"></td><td class="upload-progress {{ if UploadMultiple.Anidex.Status == 1}}pending{{else if UploadMultiple.Anidex.Status == 2}}error{{else}}finished{{end}}">{{ if UploadMultiple.Anidex.Status == 1}}{{ T("pending") }}{{else if UploadMultiple.Anidex.Status == 2}}{{ T("error") }}{{else}}{{ T("finished") }}{{end}}</td></tr>
+			<tr><td colspan="4" class="uploaded-url"><input type="text" id="anidexMess" class="form-input" value="{{UploadMultiple.Anidex.Message}}" disabled></td></tr>
+			{{end}}
+			
+			{{ if UploadMultiple.TTosho.Status != 0 }}
+			<tr><td colspan="4"><h3>Tokyo Tosho {{ T("upload_status") }}</h3></td></tr>
+			<tr class="upload-status ttosho"><td class="upload-site-name">Tokyo Tosho</td><td>{{ T("status") }}:</td><td class="statusicon icon-{{ if UploadMultiple.TTosho.Status == 1}}pending pending{{else if UploadMultiple.TTosho.Status == 2}}error error{{else}}finished finished{{end}}"></td><td class="upload-progress {{ if UploadMultiple.TTosho.Status == 1}}pending{{else if UploadMultiple.TTosho.Status == 2}}error{{else}}finished{{end}}">{{ if UploadMultiple.TTosho.Status == 1}}{{ T("pending") }}{{else if UploadMultiple.TTosho.Status == 2}}{{ T("error") }}{{else}}{{ T("finished") }}{{end}}</td></tr>
+			<tr><td colspan="4" class="uploaded-url"><input type="text" id="ttoshoMess" class="form-input" value="{{UploadMultiple.TTosho.Message}}" disabled></td></tr>
+			{{end}}
 		</tbody>
 	</table>
  
@@ -24,4 +34,38 @@
 {{ block footer_js()}}
 <script type="text/javascript" src="/js/translation.js?v={{ Config.Version}}{{ Config.Build }}"></script>
 <script type="text/javascript" src="/js/upload.js?v={{ Config.Version}}{{ Config.Build }}"></script>
+<script type="text/javascript">
+var multiUploadStatus = ["", "pending", "error", "finished"]
+T.Add("pending", "{{ T("pending") }}")
+T.Add("error", "{{ T("error") }}")
+T.Add("finished", "{{ T("finished") }}")
+function checkStatus() {
+	Query.Get(window.location.href+"?json", function(data) {
+		count = 3
+		if (data.anidex.status > 0) {
+			document.getElementById("anidexMess").value = data.anidex.message
+			document.querySelector("tr.upload-status.anidex .statusicon").className = "statusicon icon-"+multiUploadStatus[data.anidex.status]+" "+multiUploadStatus[data.anidex.status]
+			document.querySelector("tr.upload-status.anidex .upload-progress").className = "upload-progress "+multiUploadStatus[data.anidex.status]
+			document.querySelector("tr.upload-status.anidex .upload-progress").textContent = T.r(multiUploadStatus[data.anidex.status])
+		}
+		if (data.anidex.status != 1) count--
+		if (data.nyaasi.status > 0) {
+			document.getElementById("nyaasiMess").value = data.nyaasi.message
+			document.querySelector("tr.upload-status.nyaasi .statusicon").className = "statusicon icon-"+multiUploadStatus[data.nyaasi.status]+" "+multiUploadStatus[data.nyaasi.status]
+			document.querySelector("tr.upload-status.nyaasi .upload-progress").className = "upload-progress "+multiUploadStatus[data.nyaasi.status]
+			document.querySelector("tr.upload-status.nyaasi .upload-progress").textContent = T.r(multiUploadStatus[data.anidex.status])
+		}
+		if (data.anidex.status != 1) count--
+		if (data.ttosho.status > 0) {
+			document.getElementById("ttoshoMess").value = data.ttosho.message
+			document.querySelector("tr.upload-status.nyaasi .statusicon").className = "statusicon icon-"+multiUploadStatus[data.ttosho.status]+" "+multiUploadStatus[data.ttosho.status]
+			document.querySelector("tr.upload-status.nyaasi .upload-progress").className = "upload-progress "+multiUploadStatus[data.ttosho.status]
+			document.querySelector("tr.upload-status.nyaasi .upload-progress").textContent = T.r(multiUploadStatus[data.anidex.status])
+		}
+		if (data.anidex.status != 1) count--
+		if (count > 0) setTimeout(checkStatus, 5000)
+	})
+}
+checkStatus()
+</script>
 {{end}}

templates/template_test.go

@@ -9,6 +9,7 @@ import (
 	"path"
 	"testing"
 
+	"github.com/NyaaPantsu/nyaa/utils/upload"
 	"github.com/NyaaPantsu/nyaa/utils/validator/announcement"
 
 	"strings"
@@ -213,6 +214,10 @@ func walkDirTest(dir string, t *testing.T) {
 			variables.Set("Form", models.Tags{*fakeTag, *fakeTag, *fakeTag})
 			return variables
 		},
+		"upload_multiple.jet.html": func(variables jet.VarMap) jet.VarMap {
+			variables.Set("UploadMultiple", upload.MultipleForm{})
+			return variables
+		},
 	}
 
 	fmt.Printf("\nTesting Folder: %s\n", dir)

translations/CHANGELOG.md

@@ -77,6 +77,16 @@
 * + read_notifications_cleared
 * + notifications_read
 * + mark_notifications_as_read
+## 2017/10/30
+* + status
+* + upload_status
+* + error
+* + pending
+* + finished
+* + upload_to
+* + upload_to_other
+* + choose_category
+* + choose_language
 ## 2017/10/31
 * + followers
 * + comment_markdown_notice

translations/en-us.all.json

@@ -2279,6 +2279,42 @@
     "id": "status",
     "translation": "Status"
   },
+  {
+    "id": "upload_status",
+    "translation": "Upload status"
+  },
+  {
+    "id": "pending",
+    "translation": "Pending"
+  },
+  {
+    "id": "error",
+    "translation": "Error"
+  },
+  {
+    "id": "finished",
+    "translation": "Finished"
+  },
+  {
+    "id": "upload_to",
+    "translation": "Upload to"
+  },
+  {
+    "id": "upload_to_other",
+    "translation": "Upload to other websites"
+  },
+  {
+    "id": "upload_to_other",
+    "translation": "Upload to other websites"
+  },
+  {
+    "id":"choose_category",
+    "translation": "Choose a category (required)"
+  },
+  {
+    "id":"choose_language",
+    "translation":"Choose a language (required)"
+  },
   {
     "id": "event",
     "translation": "Event"

translations/fr-fr.all.json

@@ -2283,6 +2283,38 @@
     "id": "status",
     "translation": "Statut"
   },
+  {
+    "id": "upload_status",
+    "translation": "Statut de l'upload"
+  },
+  {
+    "id": "pending",
+    "translation": "En attente"
+  },
+  {
+    "id": "error",
+    "translation": "Erreur"
+  },
+  {
+    "id": "finished",
+    "translation": "Terminé"
+  },
+  {
+    "id": "upload_to",
+    "translation": "Uploader vers"
+  },
+  {
+    "id": "upload_to_other",
+    "translation": "Uploader vers d'autres sites"
+  },
+  {
+    "id": "choose_category",
+    "translation": "Choisir une catégorie (requis)"
+  },
+  {
+    "id": "choose_language",
+    "translation": "Choisir une langue (requis)"
+  },
   {
     "id": "event",
     "translation": "Événement"

utils/upload/categories.go

@@ -0,0 +1,95 @@
+package upload
+
+import (
+	"fmt"
+
+	"github.com/NyaaPantsu/nyaa/config"
+	"github.com/NyaaPantsu/nyaa/models"
+)
+
+// Convert automatically our sukebei cats to platform specified Hentai cats
+var sukebeiCategories = []map[string]string{
+	ttosho: {
+		"1_1": "12",
+		"1_2": "12",
+		"1_3": "14",
+		"1_4": "13",
+		"1_5": "4",
+		"2_1": "4",
+		"2_2": "15",
+	},
+	nyaasi: {
+		"1_1": "1_1",
+		"1_2": "1_2",
+		"1_3": "1_3",
+		"1_4": "1_4",
+		"1_5": "1_5",
+		"2_1": "2_1",
+		"2_2": "2_2",
+	},
+}
+
+var normalCategories = []map[string]string{
+	ttosho: {
+		"3_12": "1",
+		"3_5":  "1",
+		"3_13": "10",
+		"3_6":  "7",
+		"2_3":  "2",
+		"2_4":  "2",
+		"4_7":  "3",
+		"4_8":  "7",
+		"4_14": "10",
+		"5_9":  "8",
+		"5_10": "8",
+		"5_18": "10",
+		"5_11": "7",
+		"6_15": "5",
+		"6_16": "5",
+		"1_1":  "5",
+		"1_2":  "5",
+	},
+	nyaasi: {
+		"3_12": "1_1",
+		"3_5":  "1_2",
+		"3_13": "1_3",
+		"3_6":  "1_4",
+		"2_3":  "2_1",
+		"2_4":  "2_2",
+		"4_7":  "3_1",
+		"4_8":  "3_4",
+		"4_14": "3_3",
+		"5_9":  "4_1",
+		"5_10": "4_2",
+		"5_18": "4_3",
+		"5_11": "4_4",
+		"6_15": "5_1",
+		"6_16": "5_2",
+		"1_1":  "6_1",
+		"1_2":  "6_2",
+	},
+}
+
+// Category returns the category converted from nyaa one to tosho one
+func Category(platform int, t *models.Torrent) string {
+	cat := fmt.Sprintf("%d_%d", t.Category, t.SubCategory)
+	// if we are in sukebei, there are some categories
+	if config.IsSukebei() {
+		// check that platform exist in our map for sukebei categories
+		if platform < len(sukebeiCategories) {
+			// return the remaped category if it exists
+			if val, ok := sukebeiCategories[platform][cat]; ok {
+				return val
+			}
+		}
+	}
+	// check that platform exist in our map
+	if platform >= len(normalCategories) {
+		return ""
+	}
+	// return the remaped category if it exists
+	if val, ok := normalCategories[platform][cat]; ok {
+		return val
+	}
+	return ""
+}

utils/upload/categories_test.go

@@ -0,0 +1,38 @@
+package upload
+
+import (
+	"testing"
+
+	"github.com/NyaaPantsu/nyaa/config"
+	"github.com/NyaaPantsu/nyaa/models"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCategory(t *testing.T) {
+	assert := assert.New(t)
+	dummyTorrent := &models.Torrent{Category: 1, SubCategory: 1}
+	tests := []struct {
+		torrent  *models.Torrent
+		platform int
+		sukebei  bool
+		expected string
+	}{
+		{dummyTorrent, anidex, false, ""},
+		{dummyTorrent, ttosho, false, "5"},
+		{dummyTorrent, ttosho, true, "12"},
+		{dummyTorrent, nyaasi, false, "6_1"},
+		{dummyTorrent, nyaasi, true, "1_1"},
+		{dummyTorrent, 20, true, ""},
+		{&models.Torrent{Category: 33, SubCategory: 33}, nyaasi, true, ""},
+	}
+
+	for _, test := range tests {
+		if test.sukebei {
+			// workaround to make the function believe we are in sukebei
+			config.Get().Models.TorrentsTableName = "sukebei_torrents"
+		} else {
+			config.Get().Models.TorrentsTableName = "torrents"
+		}
+		assert.Equal(test.expected, Category(test.platform, test.torrent))
+	}
+}

utils/upload/errors.go

@@ -0,0 +1,6 @@
+package upload
+
+import "errors"
+
+var errPending = errors.New("Magnet being generated already")
+var errConfig = errors.New("Magnet Empty or FileStorage not configured")

utils/upload/generate.go

@@ -1,12 +1,15 @@
 package upload
 
 import (
-	"strconv"
-	"errors"
 	"fmt"
 	"os"
+	"strconv"
+	"strings"
+	"time"
 
 	"github.com/NyaaPantsu/nyaa/config"
+	"github.com/NyaaPantsu/nyaa/models"
+	"github.com/NyaaPantsu/nyaa/utils/format"
 	"github.com/NyaaPantsu/nyaa/utils/log"
 	"github.com/anacrolix/dht"
 	"github.com/anacrolix/torrent"
@@ -41,12 +44,12 @@ func GenerateTorrent(magnet string) error {
 		}
 	}
 	if magnet == "" || len(config.Get().Torrents.FileStorage) == 0 {
-		return errors.New("Magnet Empty or FileStorage not configured")
+		return errConfig
 	}
 	if len(queue) > 0 {
 		for _, m := range queue {
 			if m == magnet {
-				return errors.New("Magnet being generated already")
+				return errPending
 			}
 		}
 	}
@@ -57,28 +60,63 @@ func GenerateTorrent(magnet string) error {
 		log.Errorf("error adding magnet to client: %s", err)
 		return err
 	}
-	go func() {
+	<-t.GotInfo()
-		<-t.GotInfo()
+	mi := t.Metainfo()
-		mi := t.Metainfo()
+	t.Drop()
-		t.Drop()
+	file := fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, t.InfoHash().String())
-		file := fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, t.InfoHash().String())
+	f, err := os.Create(file)
-		f, err := os.Create(file)
+	if err != nil {
-		if err != nil {
+		log.Errorf("error creating torrent metainfo file: %s", err)
-			log.Errorf("error creating torrent metainfo file: %s", err)
+		return err
-			return
+	}
+	defer f.Close()
+	defaultTracker := config.Get().Torrents.Trackers.GetDefault()
+	if defaultTracker != "" {
+		mi.Announce = defaultTracker
+	}
+	err = bencode.NewEncoder(f).Encode(mi)
+	if err != nil {
+		log.Errorf("error writing torrent metainfo file: %s", err)
+		return err
+	}
+	for k, m := range queue {
+		if m == magnet {
+			queue = append(queue[:k], queue[k+1:]...)
 		}
-		defer f.Close()
+	}
-		err = bencode.NewEncoder(f).Encode(mi)
+	log.Infof("New torrent file generated in: %s", file)
-		if err != nil {
+
-			log.Errorf("error writing torrent metainfo file: %s", err)
+	return nil
-			return
+}
-		}
+
-		for k, m := range queue {
+// GotFile will check if a torrent file exists and if not, try to generate it
-			if m == magnet {
+func GotFile(torrent *models.Torrent) error {
-				queue = append(queue[:k], queue[k+1:]...)
+	var trackers []string
-			}
+	if torrent.Trackers == "" {
-		}
+		trackers = config.Get().Torrents.Trackers.Default
-		log.Infof("New torrent file generated in: %s", file)
+	} else {
-	}()
+		trackers = torrent.GetTrackersArray()
+	}
+	// We generate a new magnet link with all trackers (ours + ones from uploader)
+	magnet := format.InfoHashToMagnet(strings.TrimSpace(torrent.Hash), torrent.Name, trackers...)
+	//Check if file exists and open
+	_, err := os.Open(torrent.GetPath())
+	if err != nil {
+		err := GenerateTorrent(magnet)
+		if err != errPending && err != nil {
+			return err
+		}
+		if err == errPending {
+			for {
+				//Check again if file exists and open
+				_, err := os.Open(torrent.GetPath())
+				if err == nil {
+					break
+				}
+				// We check every 10 seconds
+				time.Sleep(10000)
+			}
+		}
+	}
 	return nil
 }

utils/upload/multi.go

@@ -0,0 +1,367 @@
+package upload
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/NyaaPantsu/nyaa/config"
+	"github.com/NyaaPantsu/nyaa/models"
+	"github.com/NyaaPantsu/nyaa/utils/cache"
+	"github.com/NyaaPantsu/nyaa/utils/log"
+)
+
+const (
+	anidex = iota
+	nyaasi
+	ttosho
+)
+
+const (
+	pendingState = iota + 1
+	errorState
+	doneState
+)
+
+// Each service gives a status and a message when uploading to them
+type service struct {
+	Status  int    `json:"status"`
+	Message string `json:"message"`
+}
+
+// MultipleForm is a struct used to follow the status of the uploads
+type MultipleForm struct {
+	PantsuID uint    `json:"id"`
+	Anidex   service `json:"anidex"`
+	Nyaasi   service `json:"nyaasi"`
+	TTosho   service `json:"ttosho"`
+}
+
+// ToAnidex : function to upload a torrent to anidex
+// TODO: subCat and lang should be taken from torrent model and not asked to be typed again
+// so making the conversion here would be better
+func ToAnidex(torrent *models.Torrent, apiKey string, subCat string, lang string) {
+	uploadMultiple := MultipleForm{PantsuID: torrent.ID, Anidex: service{Status: pendingState}}
+	uploadMultiple.save(anidex)
+	log.Info("Create anidex instance")
+
+	// If the torrent is posted as anonymous or apikey is not set, we set it with default value
+	if apiKey == "" || (torrent.Hidden && apiKey != "") {
+		apiKey = config.Get().Upload.DefaultAnidexToken
+	}
+
+	if apiKey == "" { // You need to check that apikey is not empty even after config. Since it is left empty in config by default and is required
+		log.Errorf("ApiKey is empty, we can't upload to anidex for torrent %d", torrent.ID)
+		uploadMultiple.updateAndSave(anidex, errorState, "No ApiKey providen (required)")
+		return
+	}
+	extraParams := map[string]string{
+		//Required
+		"api_key":   apiKey,
+		"subcat_id": subCat,
+		"group_id":  "0",
+		"lang_id":   lang,
+
+		//Optional
+		"description":  torrent.Description,
+		"torrent_name": torrent.Name,
+	}
+	if config.IsSukebei() {
+		extraParams["hentai"] = "1"
+	}
+	if torrent.IsRemake() {
+		extraParams["reencode"] = "1"
+	}
+	if torrent.IsAnon() {
+		extraParams["private"] = "1"
+	}
+	request, err := newfileUploadRequest("https://anidex.info/api/", extraParams, "file", torrent.GetPath())
+	if err != nil {
+		log.CheckError(err)
+		return
+	}
+	client := &http.Client{}
+	rsp, err := client.Do(request)
+	if err != nil {
+		log.CheckError(err)
+		return
+	}
+	log.Info("Launch anidex http request")
+
+	if err != nil {
+		uploadMultiple.updateAndSave(anidex, errorState, "Error during the HTTP POST request")
+		log.CheckErrorWithMessage(err, "Error in request: %s")
+		return
+	}
+	defer rsp.Body.Close()
+	bodyByte, err := ioutil.ReadAll(rsp.Body)
+	if err != nil {
+		uploadMultiple.updateAndSave(anidex, errorState, "Unknown error")
+		log.CheckErrorWithMessage(err, "Error in parsing request: %s")
+		return
+	}
+	if uploadMultiple.Anidex.Status == pendingState {
+		uploadMultiple.Anidex.Message = string(bodyByte)
+		if strings.Contains(uploadMultiple.Anidex.Message, "http://") {
+			uploadMultiple.Anidex.Status = doneState
+		} else {
+			uploadMultiple.Anidex.Status = errorState
+		}
+		uploadMultiple.save(anidex)
+		log.Info("Anidex request done")
+		fmt.Println(uploadMultiple)
+	}
+}
+
+// ToNyaasi : function to upload a torrent to anidex
+func ToNyaasi(username string, password string, torrent *models.Torrent) {
+	uploadMultiple := MultipleForm{PantsuID: torrent.ID, Nyaasi: service{Status: pendingState}}
+	uploadMultiple.Nyaasi.Message = "Sorry u are not allowed"
+	uploadMultiple.save(nyaasi)
+	log.Info("Create NyaaSi instance")
+
+	// If the torrent is posted as anonymous or apikey is not set, we set it with default value
+	if username == "" || (torrent.Hidden && username != "") {
+		username = config.Get().Upload.DefaultNyaasiUsername
+		password = config.Get().Upload.DefaultNyaasiPassword
+	}
+
+	if username == "" || password == "" { // You need to check that username AND password are not empty even after config. Since they are left empty in config by default and are required
+		log.Errorf("Username or Password is empty, we can't upload to Nyaa.si for torrent %d", torrent.ID)
+		uploadMultiple.updateAndSave(nyaasi, errorState, "No valid account providen (required)")
+		return
+	}
+
+	params := map[string]interface{}{
+		"name":        torrent.Name,
+		"category":    Category(nyaasi, torrent),
+		"information": "",
+		"description": torrent.Description,
+		"anonymous":   torrent.IsAnon(),
+		"hidden":      false,
+		"remake":      torrent.IsRemake(),
+		"trusted":     torrent.IsTrusted(),
+	}
+	torrentData, _ := json.Marshal(params)
+	extraParams := map[string]string{
+		"torrent_data": string(torrentData),
+	}
+
+	request, err := newfileUploadRequest("https://nyaa.si/api/upload", extraParams, "torrent", torrent.GetPath())
+	if err != nil {
+		log.CheckError(err)
+		return
+	}
+	request.SetBasicAuth(username, password)
+	client := &http.Client{}
+	rsp, err := client.Do(request)
+	if err != nil {
+		log.CheckError(err)
+		return
+	}
+	log.Info("Launch Nyaa.Si http request")
+
+	if err != nil {
+		uploadMultiple.updateAndSave(nyaasi, errorState, "Error during the HTTP POST request")
+		log.CheckErrorWithMessage(err, "Error in request: %s")
+		return
+	}
+	defer rsp.Body.Close()
+	bodyByte, err := ioutil.ReadAll(rsp.Body)
+	if err != nil {
+		uploadMultiple.updateAndSave(nyaasi, errorState, "Unknown error")
+		log.CheckErrorWithMessage(err, "Error in parsing request: %s")
+		return
+	}
+	if uploadMultiple.Nyaasi.Status == pendingState {
+		var data map[string]interface{}
+		if err = json.Unmarshal(bodyByte, &data); err != nil {
+			log.CheckErrorWithMessage(err, "Cannot unmarshal json Response after upload request to Nyaa.Si")
+			uploadMultiple.Nyaasi.Status = errorState
+			uploadMultiple.Nyaasi.Message = err.Error()
+		}
+		if _, ok := data["errors"]; ok {
+			uploadMultiple.Nyaasi.Status = errorState
+			uploadMultiple.Nyaasi.Message = string(bodyByte)
+		} else {
+			uploadMultiple.Nyaasi.Status = doneState
+			uploadMultiple.Nyaasi.Message = fmt.Sprintf("%s", data["url"].(string))
+		}
+		uploadMultiple.save(nyaasi)
+		log.Info("Nyaa.Si request done")
+		fmt.Println(uploadMultiple)
+	}
+}
+
+// ToTTosho : function to upload a torrent to TokyoTosho
+func ToTTosho(apiKey string, torrent *models.Torrent) {
+	uploadMultiple := MultipleForm{PantsuID: torrent.ID, TTosho: service{Status: pendingState}}
+	uploadMultiple.save(ttosho)
+	log.Info("Create TokyoTosho instance")
+
+	// If the torrent is posted as anonymous or apikey is not set, we set it with default value
+	if apiKey == "" || (torrent.Hidden && apiKey != "") {
+		apiKey = config.Get().Upload.DefaultTokyoTToken
+	}
+
+	if apiKey == "" { // You need to check that apikey is not empty even after config. Since it is left empty in config by default and is required
+		log.Errorf("ApiKey is empty, we can't upload to TokyoTosho for torrent %d", torrent.ID)
+		uploadMultiple.updateAndSave(ttosho, errorState, "No ApiKey providen (required)")
+		return
+	}
+
+	extraParams := map[string]string{
+		//Required
+		"apikey": apiKey,
+		"url":    torrent.Download(),
+		"type":   Category(ttosho, torrent),
+		"send":   "true",
+
+		//Optional
+		"website": torrent.WebsiteLink,
+		"comment": torrent.Description,
+	}
+	request, err := newUploadRequest("https://www.tokyotosho.info/new.php", extraParams)
+	if err != nil {
+		log.CheckError(err)
+		return
+	}
+	client := &http.Client{}
+	rsp, err := client.Do(request)
+	if err != nil {
+		log.CheckError(err)
+		return
+	}
+	log.Info("Launch TokyoTosho http request")
+
+	if err != nil {
+		uploadMultiple.updateAndSave(ttosho, errorState, "Error during the HTTP POST request")
+		log.CheckErrorWithMessage(err, "Error in request: %s")
+		return
+	}
+	defer rsp.Body.Close()
+	bodyByte, err := ioutil.ReadAll(rsp.Body)
+	if err != nil {
+		uploadMultiple.updateAndSave(ttosho, errorState, "Unknown error")
+		log.CheckErrorWithMessage(err, "Error in parsing request: %s")
+		return
+	}
+	if uploadMultiple.TTosho.Status == pendingState {
+		if strings.Contains(string(bodyByte), "OK,") {
+			uploadMultiple.TTosho.Status = doneState
+			idnumber := strings.Split(string(bodyByte), ",")
+			uploadMultiple.TTosho.Message = fmt.Sprintf("Upload done! https://www.tokyotosho.info/details.php?id=%s", idnumber[1])
+		} else {
+			uploadMultiple.TTosho.Status = errorState
+			uploadMultiple.TTosho.Message = string(bodyByte)
+		}
+		uploadMultiple.save(ttosho)
+		log.Info("TokyoTosho request done")
+		fmt.Println(uploadMultiple)
+	}
+}
+
+// Saves the multipleform in each go routines and share the state of each upload for 5 minutes
+// After timeout, the multipleform is flushed from memory
+func (u *MultipleForm) save(which int) {
+	// We check if there is already a variable shared, if there is we update only the status/message of one service
+	if found, ok := cache.C.Get(fmt.Sprintf("tstatus_%d", u.PantsuID)); ok {
+		uploadStatus := found.(MultipleForm)
+		switch which {
+		case anidex:
+			uploadStatus.Anidex = u.Anidex
+			break
+		case nyaasi:
+			uploadStatus.Nyaasi = u.Nyaasi
+			break
+		case ttosho:
+			uploadStatus.TTosho = u.TTosho
+			break
+		}
+		u = &uploadStatus
+	}
+	// And then we save the variable in cache
+	cache.C.Set(fmt.Sprintf("tstatus_%d", u.PantsuID), *u, 5*time.Minute)
+}
+
+// shortcut function to update and save a service
+func (u *MultipleForm) updateAndSave(which int, code int, message string) {
+	switch which {
+	case anidex:
+		u.Anidex.update(code, message)
+		break
+	case nyaasi:
+		u.Nyaasi.update(code, message)
+		break
+	case ttosho:
+		u.TTosho.update(code, message)
+		break
+	}
+	u.save(which)
+}
+
+// shortcut function to update both code and message of a service
+func (s *service) update(code int, message string) {
+	s.Status = code
+	s.Message = message
+}
+
+// Creates a new file upload http request with optional extra params
+func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	fi, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	body := new(bytes.Buffer)
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile(paramName, fi.Name())
+	if err != nil {
+		return nil, err
+	}
+	if _, err = io.Copy(part, file); err != nil {
+		return nil, err
+	}
+	for key, val := range params {
+		_ = writer.WriteField(key, val)
+	}
+	err = writer.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	request, err := http.NewRequest("POST", uri, body)
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Add("Content-Type", writer.FormDataContentType())
+	return request, nil
+}
+
+// Creates a new upload http request with optional extra params
+func newUploadRequest(uri string, params map[string]string) (*http.Request, error) {
+	var form url.Values
+	for key, val := range params {
+		form[key] = append(form[key], val)
+	}
+
+	body := bytes.NewBufferString(form.Encode())
+	request, err := http.NewRequest("POST", uri, body)
+	if err != nil {
+		return nil, err
+	}
+	return request, nil
+}

utils/upload/upload.go

@@ -1,11 +1,6 @@
 package upload
 
 import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"mime/multipart"
-	"os"
 	"reflect"
 	"strings"
 
@@ -123,7 +118,7 @@ func ExtractInfo(c *gin.Context, r *torrentValidator.TorrentRequest) error {
 		return err
 	}
 
-	tfile, err := r.ValidateMultipartUpload(c, uploadFormTorrent)
+	err = r.ValidateMultipartUpload(c, uploadFormTorrent)
 	if err != nil {
 		return err
 	}
@@ -133,16 +128,8 @@ func ExtractInfo(c *gin.Context, r *torrentValidator.TorrentRequest) error {
 	if err != nil {
 		return err
 	}
-
+	// We are not saving the file here because we need to add our own tracker list to the torrent file, therefore, we generate the torrent file at upload time through GenerateTorrent()
-	// after data has been checked & extracted, write it to disk
+	// when it is magnet
-	if len(config.Get().Torrents.FileStorage) > 0 && r.Filesize > 0 {
-		err := writeTorrentToDisk(tfile, r.Infohash+".torrent", &r.Filepath)
-		if err != nil {
-			return err
-		}
-	} else {
-		r.Filepath = ""
-	}
 
 	return nil
 }
@@ -213,19 +200,6 @@ func UpdateUnscopeTorrent(r *torrentValidator.UpdateRequest, t *models.Torrent,
 	return t
 }
 
-func writeTorrentToDisk(file multipart.File, name string, fullpath *string) error {
-	_, seekErr := file.Seek(0, io.SeekStart)
-	if seekErr != nil {
-		return seekErr
-	}
-	b, err := ioutil.ReadAll(file)
-	if err != nil {
-		return err
-	}
-	*fullpath = fmt.Sprintf("%s%c%s", config.Get().Torrents.FileStorage, os.PathSeparator, name)
-	return ioutil.WriteFile(*fullpath, b, 0644)
-}
-
 // NewTorrentRequest : creates a new torrent request struc with some default value
 func NewTorrentRequest(params ...string) *torrentValidator.TorrentRequest {
 	torrentRequest := &torrentValidator.TorrentRequest{}

utils/validator/torrent/errors.go

@@ -15,3 +15,4 @@ var errTorrentHashInvalid = errors.New("torrent_hash_invalid")
 var errTorrentTagsInvalid = errors.New("torrent_tags_invalid")
 var errMissingFieldConfig = errors.New("Torrent Configuration in config.yml is invalid, missing a 'field' attribute in tags types")
 var errWrongFieldConfig = errors.New("Torrent Configuration in config.yml is invalid, wrong 'field' attribute in tags types")
+var errInvalidTorrentFile = errors.New("torrent_file_invalid")

utils/validator/torrent/functions.go

@@ -4,9 +4,8 @@ import (
 	"encoding/base32"
 	"encoding/hex"
 	"fmt"
-	"io"
-	"mime/multipart"
 	"net/url"
+	"os"
 	"reflect"
 	"regexp"
 	"strconv"
@@ -15,13 +14,11 @@ import (
 	"github.com/NyaaPantsu/nyaa/config"
 	"github.com/NyaaPantsu/nyaa/utils/categories"
 	"github.com/NyaaPantsu/nyaa/utils/cookies"
-	"github.com/NyaaPantsu/nyaa/utils/format"
 	msg "github.com/NyaaPantsu/nyaa/utils/messages"
-	"github.com/NyaaPantsu/nyaa/utils/metainfo"
 	"github.com/NyaaPantsu/nyaa/utils/torrentLanguages"
 	"github.com/NyaaPantsu/nyaa/utils/validator/tag"
+	"github.com/anacrolix/torrent/metainfo"
 	"github.com/gin-gonic/gin"
-	"github.com/zeebo/bencode"
 )
 
 // ValidateName is a function validating the torrent name
@@ -191,80 +188,90 @@ func (r *TorrentRequest) ExtractLanguage() error {
 }
 
 // ValidateMultipartUpload : Check if multipart upload is valid
-func (r *TorrentRequest) ValidateMultipartUpload(c *gin.Context, uploadFormTorrent string) (multipart.File, error) {
+func (r *TorrentRequest) ValidateMultipartUpload(c *gin.Context, uploadFormTorrent string) error {
 	// first: parse torrent file (if any) to fill missing information
 	tfile, _, err := c.Request.FormFile(uploadFormTorrent)
 	if err == nil {
-		var torrent metainfo.TorrentFile
+		torrent, err := metainfo.Load(tfile)
-
-		// decode torrent
-		_, seekErr := tfile.Seek(0, io.SeekStart)
-		if seekErr != nil {
-			return tfile, seekErr
-		}
-		err = bencode.NewDecoder(tfile).Decode(&torrent)
 		if err != nil {
-			return tfile, metainfo.ErrInvalidTorrentFile
+			return err
+		}
+
+		torrentInfos, err := torrent.UnmarshalInfo()
+		if err != nil {
+			return err
 		}
 
 		// check a few things
-		if torrent.IsPrivate() {
+		if torrentInfos.Private != nil && *torrentInfos.Private {
-			return tfile, errTorrentPrivate
+			return errTorrentPrivate
 		}
-		trackers := torrent.GetAllAnnounceURLS()
+		// We check that the trackers are valid,
-		r.Trackers = CheckTrackers(trackers)
+		// we filter the dead ones
+		// and we add the needed ones from our config in the request and the torrent file
+		r.Trackers = CheckTrackers(torrent)
 		if len(r.Trackers) == 0 {
-			return tfile, errTorrentNoTrackers
+			return errTorrentNoTrackers
 		}
 
-		// Name
+		// For the name, if none is provided in request, we set the name from the torrent file
 		if len(r.Name) == 0 {
-			r.Name = torrent.TorrentName()
+			r.Name = torrentInfos.Name
 		}
 
 		// Magnet link: if a file is provided it should be empty
 		if len(r.Magnet) != 0 {
-			return tfile, errTorrentAndMagnet
+			return errTorrentAndMagnet
 		}
 
-		_, seekErr = tfile.Seek(0, io.SeekStart)
+		// We are using here default functions from anacrolix
-		if seekErr != nil {
+		infohash := torrent.HashInfoBytes()
-			return tfile, seekErr
-		}
-		infohash, err := metainfo.DecodeInfohash(tfile)
-		if err != nil {
-			return tfile, metainfo.ErrInvalidTorrentFile
-		}
-		r.Infohash = infohash
-		r.Magnet = format.InfoHashToMagnet(infohash, r.Name, trackers...)
 
+		if infohash.String() == "" {
+			return errInvalidTorrentFile
+		}
+		r.Infohash = infohash.String()
+		r.Magnet = torrent.Magnet(r.Name, infohash).String()
 		// extract filesize
-		r.Filesize = int64(torrent.TotalSize())
+		r.Filesize = torrentInfos.TotalLength()
 
 		// extract filelist
-		fileInfos := torrent.Info.GetFiles()
+		fileInfos := torrentInfos.Files
 		for _, fileInfo := range fileInfos {
 			r.FileList = append(r.FileList, uploadedFile{
 				Path:     fileInfo.Path,
-				Filesize: int64(fileInfo.Length),
+				Filesize: fileInfo.Length,
 			})
 		}
+		// Since we can change with anacrolix the trackers of a torrent,
+		// We will use this to generate directly the torrent file here
+		file := fmt.Sprintf("%s%c%s.torrent", config.Get().Torrents.FileStorage, os.PathSeparator, infohash.String())
+		f, err := os.Create(file)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		err = torrent.Write(f)
+		if err != nil {
+			return err
+		}
+
 	} else {
 		err = r.ValidateMagnet()
 		if err != nil {
-			return tfile, err
+			return err
 		}
 		err = r.ValidateHash()
 		if err != nil {
-			return tfile, err
+			return err
 		}
 		// TODO: Get Trackers from magnet URL
 		r.Filesize = 0
 		r.Filepath = ""
 
-		return tfile, nil
+		return nil
 	}
-	return tfile, err
+	return err
 }
 
 // ExtractInfo : Function to assign values from request to ReassignForm

utils/validator/torrent/helpers.go

@@ -3,40 +3,61 @@ package torrentValidator
 import (
 	"net/url"
 	"strings"
+
+	"github.com/NyaaPantsu/nyaa/config"
+	"github.com/anacrolix/torrent/metainfo"
 )
 
 // CheckTrackers : Check if there is good trackers in torrent
-func CheckTrackers(trackers []string) []string {
+func CheckTrackers(t *metainfo.MetaInfo) []string {
 	// TODO: move to runtime configuration
-	var deadTrackers = []string{ // substring matches!
+	var deadTrackers = config.Get().Torrents.Trackers.DeadTrackers
-		"://open.nyaatorrents.info:6544",
-		"://tracker.openbittorrent.com:80",
-		"://tracker.publicbt.com:80",
-		"://stats.anisource.net:2710",
-		"://exodus.desync.com",
-		"://open.demonii.com:1337",
-		"://tracker.istole.it:80",
-		"://tracker.ccc.de:80",
-		"://bt2.careland.com.cn:6969",
-		"://announce.torrentsmd.com:8080",
-		"://open.demonii.com:1337",
-		"://tracker.btcake.com",
-		"://tracker.prq.to",
-		"://bt.rghost.net"}
 
-	var trackerRet []string
+	trackerRet := []string{}
-	for _, t := range trackers {
+	tempList := metainfo.AnnounceList{}
-		urlTracker, err := url.Parse(t)
+	for _, group := range t.AnnounceList {
-		if err == nil {
+		var trackers []string
-			good := true
+		for _, tracker := range group {
-			for _, check := range deadTrackers {
+			urlTracker, err := url.ParseRequestURI(tracker)
-				if strings.Contains(t, check) {
+			if err == nil {
-					good = false
+				good := true
-					break // No need to continue the for loop
+				for _, check := range deadTrackers {
+					// the tracker is part of the deadtracker list
+					// we don't keep it
+					if strings.Contains(tracker, check) {
+						good = false
+						break // No need to continue the for loop
+					}
+				}
+				if good {
+					// We only keep the good trackers
+					trackers = append(trackers, urlTracker.String())
 				}
 			}
-			if good {
+		}
-				trackerRet = append(trackerRet, urlTracker.String())
+		if len(trackers) > 0 {
+			tempList = append(tempList, trackers)
+			trackerRet = append(trackerRet, trackers...)
+		}
+	}
+	t.AnnounceList = tempList
+	defaultTracker := config.Get().Torrents.Trackers.GetDefault()
+	if defaultTracker != "" {
+		t.Announce = defaultTracker
+	}
+
+	for _, key := range config.Get().Torrents.Trackers.NeededTrackers {
+		inside := false
+		if key < len(config.Get().Torrents.Trackers.Default) {
+			tracker := config.Get().Torrents.Trackers.Default[key]
+			for _, tr := range trackerRet {
+				if tr == tracker {
+					inside = true
+				}
+			}
+			if !inside {
+				trackerRet = append(trackerRet, tracker)
+				t.AnnounceList = append(t.AnnounceList, []string{tracker})
 			}
 		}
 	}

utils/validator/torrent/helpers_test.go

@@ -0,0 +1,46 @@
+package torrentValidator
+
+import (
+	"testing"
+
+	"github.com/anacrolix/torrent/metainfo"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCheckTrackers(t *testing.T) {
+	assert := assert.New(t)
+	tests := []struct {
+		Info         metainfo.MetaInfo
+		Expected     []string
+		ExpectedList metainfo.AnnounceList
+	}{
+		{
+			metainfo.MetaInfo{
+				AnnounceList: [][]string{{"udp://nn.fof:4545/ano", "lol"}, {"://nyaa.tr/ann"}, {".co", "http://mont.co:444/ann"}},
+			},
+			[]string{"udp://nn.fof:4545/ano", "http://mont.co:444/ann", "udp://tracker.uw0.xyz:6969/announce", "http://anidex.moe:6969/announce"},
+			[][]string{{"udp://nn.fof:4545/ano"}, {"http://mont.co:444/ann"}, {"udp://tracker.uw0.xyz:6969/announce"}, {"http://anidex.moe:6969/announce"}},
+		},
+		{
+			metainfo.MetaInfo{
+				AnnounceList: [][]string{{"http://open.nyaatorrents.info:6544", "http://mont.co:444/ann"}}, // dead tracker
+			},
+			[]string{"http://mont.co:444/ann", "udp://tracker.uw0.xyz:6969/announce", "http://anidex.moe:6969/announce"},
+			[][]string{{"http://mont.co:444/ann"}, {"udp://tracker.uw0.xyz:6969/announce"}, {"http://anidex.moe:6969/announce"}},
+		},
+		{
+			metainfo.MetaInfo{
+				AnnounceList: [][]string{{"http://open.nyaatorrents.info:6544", "http://mont.co:444/ann", "http://mont.co:4434/ann"}, {"http://mont.co:444/anno"}}, // dead tracker
+			},
+			[]string{"http://mont.co:444/ann", "http://mont.co:4434/ann", "http://mont.co:444/anno", "udp://tracker.uw0.xyz:6969/announce", "http://anidex.moe:6969/announce"},
+			[][]string{{"http://mont.co:444/ann", "http://mont.co:4434/ann"}, {"http://mont.co:444/anno"}, {"udp://tracker.uw0.xyz:6969/announce"}, {"http://anidex.moe:6969/announce"}},
+		},
+	}
+
+	for _, test := range tests {
+		trackers := CheckTrackers(&test.Info)
+		assert.Equal(test.Expected, trackers)
+		assert.Equal(test.ExpectedList, test.Info.AnnounceList)
+	}
+
+}