On-demand stats fetching (#1621)

* Update view.jet.html

* Update router.go

* Create stats.go

* Update view.jet.html

* Update stats.go

* Update stats.go

* rollback

* returns -1 by default

* Update stats.go

* Update stats.go

* Import goscrape & torrent

* copypaste stats.go from scrapers

* travis

* Update stats.go

* Update stats.go

* Put fresh stats into a <span>

* Background-color for freshly fetched stats

* ditto but for tomorrow

* "Loading..." text in case it gets a bit too long

* Stat fetching in controller (missing DB update)

* Update last scraped date too

* update torrent.Scrape tho it doesn't  do anything

* forgot to edit a name

* Update scrape.go

* Update stats.go

* Update stats.go

* flush cache to update stats in listing

* Change function name

* Update stats.go

* Update stats.go

* fix travis

* Update view.jet.html

* Update stats.go

* Update stats.go

* Update stats.go

* Update stats.go

* Update stats.go

.travis.yml

@@ -9,6 +9,8 @@ before_install:
 - go get github.com/go-playground/overalls
 - go get golang.org/x/tools/cmd/cover
 - go get github.com/mattn/goveralls
+- go get github.com/Stephen304/goscrape
+- go get github.com/anacrolix/torrent
 before_script:
 - go vet
 - go test -v ./...

controllers/torrent/router.go

@@ -7,6 +7,7 @@ import (
 
 func init() {
 	router.Get().Any("/download/:hash", DownloadTorrent)
+	router.Get().Any("/stats/:id", GetStatsHandler)
 
 	torrentRoutes := router.Get().Group("/torrent", middlewares.LoggedInMiddleware())
 	{

controllers/torrent/stats.go

@@ -0,0 +1,65 @@
+package torrentController
+
+import (
+	"text/template"
+	"strconv"
+	"strings"
+	"net/url"
+	"time"
+	"fmt"
+
+	"github.com/NyaaPantsu/nyaa/models/torrents"
+	"github.com/NyaaPantsu/nyaa/models"
+	"github.com/Stephen304/goscrape"
+	"github.com/gin-gonic/gin"
+)
+
+// ViewHeadHandler : Controller for getting torrent stats
+func GetStatsHandler(c *gin.Context) {
+	id, err := strconv.ParseInt(c.Param("id"), 10, 32)
+	if err != nil {
+		return
+	}
+
+	torrent, err := torrents.FindRawByID(uint(id))
+
+	if err != nil {
+		return
+	}
+	
+	var Trackers []string
+	for _, line := range strings.Split(torrent.Trackers[3:], "&tr=") {
+		tracker, error := url.QueryUnescape(line)
+		if error == nil && tracker[:6] == "udp://" {
+			Trackers = append(Trackers, tracker)
+		}
+	}	
+
+	scraper := goscrape.NewBulk(Trackers)
+	
+	stats := scraper.ScrapeBulk([]string{
+	  torrent.Hash,
+	})[0]
+	
+	emptyStats := goscrape.Result{stats.Btih, 0, 0, 0} 
+	
+	if stats == emptyStats {
+		stats.Seeders = -1
+		//If we put seeders on -1, the script instantly knows the fetching did not give any result, avoiding having to check all three stats below and in view.jet.html's javascript
+	}
+	
+	t, err := template.New("foo").Parse(fmt.Sprintf(`{{define "stats"}}{ "seeders": [%d], "leechers": [%d], "downloads": [%d] }{{end}}`, stats.Seeders, stats.Leechers, stats.Completed))
+	t.ExecuteTemplate(c.Writer, "stats", "")
+	
+	if stats.Seeders != -1 {
+		var tmp models.Scrape
+		if models.ORM.Where("torrent_id = ?", id).Find(&tmp).RecordNotFound() {
+			torrent.Scrape = torrent.Scrape.Create(uint(id), uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now())
+		} else {
+			torrent.Scrape = &models.Scrape{uint(id), uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now()}
+			torrent.Scrape.Update(false)
+		}
+	}
+	
+	return
+}

models/scrape.go

@@ -1,9 +1,15 @@
 package models
 
 import (
+	"net/http"
+	"errors"
 	"time"
+	"fmt"
 
 	"github.com/NyaaPantsu/nyaa/config"
+	"github.com/NyaaPantsu/nyaa/utils/cache"
+	"github.com/NyaaPantsu/nyaa/utils/log"
+	"github.com/fatih/structs"
 )
 
 // Scrape model
@@ -19,3 +25,52 @@ type Scrape struct {
 func (t Scrape) TableName() string {
 	return config.Get().Models.ScrapeTableName
 }
+
+// Update : Update scrape data based on Scrape model
+func (s *Scrape) Update(unscope bool) (int, error) {
+	db := ORM
+	if unscope {
+		db = ORM.Unscoped()
+	}
+
+	if db.Model(s).UpdateColumn(s.toMap()).Error != nil {
+		return http.StatusInternalServerError, errors.New("Scrape data was not updated")
+	}
+
+	// We only flush cache after update
+	cache.C.Delete(s.Identifier())
+	cache.C.Flush()
+
+	return http.StatusOK, nil
+}
+
+
+// toMap : convert the model to a map of interface
+func (s *Scrape) toMap() map[string]interface{} {
+	return structs.Map(s)
+}
+
+// Identifier : Return the identifier of a torrent
+func (s *Scrape) Identifier() string {
+	return fmt.Sprintf("torrent_%d", s.TorrentID)
+}
+
+//Create a Scrape entry in the DB
+func (s *Scrape) Create(torrentid uint, seeders uint32, leechers uint32, completed uint32, lastscrape time.Time) (*Scrape) {
+	ScrapeData := Scrape{
+		TorrentID:      torrentid,
+		Seeders:    	seeders,
+		Leechers: 	leechers,
+		Completed:      completed,
+		LastScrape:     lastscrape}
+
+	err := ORM.Create(&ScrapeData).Error
+	log.Infof("Scrape data ID %d created!\n", ScrapeData.TorrentID)
+	if err != nil {
+		log.CheckErrorWithMessage(err, "ERROR_SCRAPE_CREATE: Cannot create a scrape data")
+	}
+
+	ScrapeData.Update(false)
+
+	return &ScrapeData
+}

public/css/main.css

@@ -2155,6 +2155,11 @@ table.multiple-upload {
 	width: calc(100% - 30px);
 }
 
+.torrent-info-row .tr-se span, .torrent-info-row .tr-le span, .torrent-info-row .tr-dl span {
+    font-weight: bold;
+    background-color: #ffe0bf;
+}
+
 /* Language specific CSS */
 
 html[lang="ja-jp"] .form-refine span.spacing {

public/css/themes/tomorrow.css

@@ -319,3 +319,7 @@ span.tag {
     border-color: #0c0d0e;
     background: #333438;
 }
+
+.torrent-info-row .tr-se span, .torrent-info-row .tr-le span, .torrent-info-row .tr-dl span {
+    background-color: #544433;
+}

templates/site/torrents/view.jet.html

@@ -271,6 +271,34 @@
 {{end}}
 {{ block footer_js()}}
 <script type="text/javascript" src="{{ URL.Parse("/js/modal.js") }}"></script>
+{{if Torrent.LastScrape.IsZero || formatDateRFC(Torrent.LastScrape) == "0001-01-01T00:00:00Z"}}
+<script type="text/javascript">
+  var seeders   = document.querySelector(".tr-se"),
+      leechers  = document.querySelector(".tr-le"),
+      downloads = document.querySelector(".tr-dl b"),
+      torrentID = window.location.pathname.substr(6),
+	  scrapeDate= document.querySelector(".scrape-date")
+	  
+  var oldStats = [seeders.innerHTML, leechers.innerHTML, downloads.innerHTML]
+	  
+      seeders.innerHTML = "Loading..."
+      leechers.innerHTML = "Loading..."
+      downloads.innerHTML = "Loading..."
+
+  Query.Get('/stats/' + torrentID, function (data) {
+    if(typeof data.seeders != "undefined" && data.seeders != -1) {
+      seeders.innerHTML = "<span>" + data.seeders + "</span>"
+      leechers.innerHTML = "<span>" +data.leechers + "</span>"
+      downloads.innerHTML = "<span>" +data.downloads + "</span>"
+	  scrapeDate.innerText = new Date().toLocaleString(document.getElementsByTagName("html")[0].getAttribute("lang"), "ymdOpt")
+    } else {
+      seeders.innerHTML = oldStats[0]
+      leechers.innerHTML = oldStats[1]
+      downloads.innerHTML = oldStats[2]
+    }
+  })
+</script>
+{{end}}
 {{ if User.ID > 0 }}
 <script type="text/javascript" src="{{ URL.Parse("/js/template.js") }}"></script>
 <script type="text/javascript" src="{{ URL.Parse("/js/modal.js") }}"></script>