diff --git a/.gitignore b/.gitignore index ece147f5..8248a38a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ *.db main nyaa -nyaa-master.exe \ No newline at end of file +nyaa.exe +nyaa-master.exe +*.zip diff --git a/.travis.yml b/.travis.yml index aae111c2..6815c806 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: go go: - 1.x install: +- go get github.com/gorilla/feeds - go get github.com/gorilla/mux - go get github.com/mattn/go-sqlite3 - go get github.com/jinzhu/gorm diff --git a/README.md b/README.md index 13345bb5..f947ea79 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,23 @@ that anyone will be able to deploy locally or remotely. # Requirements * Golang +# Installation +* Install golang +* go get github.com/ewhal/nyaa +* go build +* Download db +* place db in folder as "nyaa.d" +* ./nyaa +* go to localhost:9999 ## TODO -* RSS feeds -* status searching -* torrent sorting +* RSS feeds(work in progress) +* torrent sorting (work in progress) * Merge 5th of april dump with animetosho dump * API improvement * Site theme -* improve search by making it less strict and support non-english characters. -* Torrent view and description page +* improve search by making it less strict +* Torrent view and description page(work in progress) * accounts? * Adding new torrents * scraping diff --git a/main.go b/main.go index 0420657f..2743602c 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,16 @@ package main import ( + "bytes" + "compress/zlib" "encoding/json" + "github.com/gorilla/feeds" "github.com/gorilla/mux" "github.com/jinzhu/gorm" _ "github.com/jinzhu/gorm/dialects/sqlite" "html" "html/template" + "io/ioutil" "log" "net/http" "strconv" @@ -16,7 +20,6 @@ import ( var db *gorm.DB var router *mux.Router - var debugLogger *log.Logger var trackers = "&tr=udp://zer0day.to:1337/announce&tr=udp://tracker.leechers-paradise.org:6969&tr=udp://explodie.org:6969&tr=udp://tracker.opentrackr.org:1337&tr=udp://tracker.coppersurfer.tk:6969" @@ -24,8 +27,7 @@ func getDBHandle() *gorm.DB { dbInit, err := gorm.Open("sqlite3", "./nyaa.db") // Migrate the schema of Torrents - // dbInit.AutoMigrate(&Torrents{}) - // dbInit.AutoMigrate(&Sub_Categories{}) + dbInit.AutoMigrate(&Torrents{}, &Categories{}, &Sub_Categories{}, &Statuses{}) checkErr(err) return dbInit @@ -37,6 +39,20 @@ func checkErr(err error) { } } +func unZlib(description []byte) string { + if (len(description) > 0) { + b := bytes.NewReader(description) + log.Println(b) + z, err := zlib.NewReader(b) + checkErr(err) + defer z.Close() + p, err := ioutil.ReadAll(z) + checkErr(err) + return string(p) + } + return "" +} + func apiHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -64,7 +80,7 @@ func apiHandler(w http.ResponseWriter, r *http.Request) { } } -func singleapiHandler(w http.ResponseWriter, r *http.Request) { +func apiViewHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id := vars["id"] @@ -100,6 +116,9 @@ func searchHandler(w http.ResponseWriter, r *http.Request) { searchQuery := r.URL.Query().Get("q") cat := r.URL.Query().Get("c") stat := r.URL.Query().Get("s") + sort := r.URL.Query().Get("sort") + order := r.URL.Query().Get("order") + catsSplit := strings.Split(cat, "_") // need this to prevent out of index panics var searchCatId, searchSubCatId string @@ -108,13 +127,21 @@ func searchHandler(w http.ResponseWriter, r *http.Request) { searchCatId = html.EscapeString(catsSplit[0]) searchSubCatId = html.EscapeString(catsSplit[1]) } + if sort == "" { + sort = "torrent_id" + } + if order == "" { + order = "desc" + } + order_by := sort + " " + order nbTorrents := 0 b := []TorrentsJson{} - torrents, nbTorrents := getTorrents(createWhereParams("torrent_name LIKE ? AND status_id LIKE ? AND category_id LIKE ? AND sub_category_id LIKE ?", - "%"+searchQuery+"%", stat+"%", searchCatId+"%", searchSubCatId+"%"), maxPerPage, maxPerPage*(pagenum-1)) + parameters := createWhereParams("torrent_name LIKE ? AND status_id LIKE ? AND category_id LIKE ? AND sub_category_id LIKE ?", + "%"+searchQuery+"%", stat+"%", searchCatId+"%", searchSubCatId+"%") + torrents, nbTorrents := getTorrentsOrderBy(¶meters, order_by, maxPerPage, maxPerPage*(pagenum-1)) for i, _ := range torrents { res := torrents[i].toJson() @@ -122,7 +149,8 @@ func searchHandler(w http.ResponseWriter, r *http.Request) { } navigationTorrents := Navigation{nbTorrents, maxPerPage, pagenum, "search_page"} - htv := HomeTemplateVariables{b, getAllCategories(false), searchQuery, stat, cat, navigationTorrents, r.URL, mux.CurrentRoute(r)} + htv := HomeTemplateVariables{b, getAllCategories(false), searchQuery, stat, cat, sort, order, navigationTorrents, r.URL, mux.CurrentRoute(r)} + err := templates.ExecuteTemplate(w, "index.html", htv) if err != nil { @@ -135,7 +163,71 @@ func safe(s string) template.URL { func faqHandler(w http.ResponseWriter, r *http.Request) { var templates = template.Must(template.New("FAQ").Funcs(funcMap).ParseFiles("templates/index.html", "templates/FAQ.html")) - err := templates.ExecuteTemplate(w, "index.html", FaqTemplateVariables{r.URL, mux.CurrentRoute(r), "", "", "", Navigation{}}) + err := templates.ExecuteTemplate(w, "index.html", FaqTemplateVariables{r.URL, mux.CurrentRoute(r), "", "", "_", "torrent_id", "desc", Navigation{}}) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func rssHandler(w http.ResponseWriter, r *http.Request) { + //vars := mux.Vars(r) + //category := vars["c"] + + // db params url + //maxPerPage := 50 // default Value maxPerPage + + torrents := getFeeds() + created := time.Now().String() + if len(torrents) > 0 { + created = torrents[0].Timestamp + } + created_as_time, err := time.Parse("2006-01-02 15:04:05", created) + if err == nil { + + } + feed := &feeds.Feed{ + Title: "Nyaa Pantsu", + Link: &feeds.Link{Href: "https://nyaa.pantsu.cat/"}, + Created: created_as_time, + } + feed.Items = []*feeds.Item{} + feed.Items = make([]*feeds.Item, len(torrents)) + + for i, _ := range torrents { + timestamp_as_time, err := time.Parse("2006-01-02 15:04:05", torrents[i].Timestamp) + if err == nil { + feed.Items[i] = &feeds.Item{ + // need a torrent view first + //Id: URL + torrents[i].Hash, + Title: torrents[i].Name, + Link: &feeds.Link{Href: string(torrents[i].Magnet)}, + Description: "", + Created: timestamp_as_time, + Updated: timestamp_as_time, + } + } + } + + rss, err := feed.ToRss() + if err == nil { + w.Write([]byte(rss)) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} +func viewHandler(w http.ResponseWriter, r *http.Request) { + var templates = template.Must(template.ParseFiles("templates/index.html", "templates/view.html")) + vars := mux.Vars(r) + id := vars["id"] + b := []TorrentsJson{} + + torrent, err := getTorrentById(id) + res := torrent.toJson() + b = append(b, res) + + htv := HomeTemplateVariables{b, getAllCategories(false), "", "", "_", "", "", Navigation{}, r.URL, mux.CurrentRoute(r)} + + err = templates.ExecuteTemplate(w, "index.html", htv) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -165,7 +257,7 @@ func rootHandler(w http.ResponseWriter, r *http.Request) { } navigationTorrents := Navigation{nbTorrents, maxPerPage, pagenum, "search_page"} - htv := HomeTemplateVariables{b, getAllCategories(false), "", "", "_", navigationTorrents, r.URL, mux.CurrentRoute(r)} + htv := HomeTemplateVariables{b, getAllCategories(false), "", "", "_", "torrent_id", "desc", navigationTorrents, r.URL, mux.CurrentRoute(r)} err := templates.ExecuteTemplate(w, "index.html", htv) if err != nil { @@ -191,9 +283,12 @@ func main() { router.HandleFunc("/page/{page:[0-9]+}", rootHandler).Name("home_page") router.HandleFunc("/search", searchHandler).Name("search") router.HandleFunc("/search/{page}", searchHandler).Name("search_page") - router.HandleFunc("/api/{page:[0-9]+}", apiHandler).Methods("GET") - router.HandleFunc("/api/torrent/{id:[0-9]+}", singleapiHandler).Methods("GET") + router.HandleFunc("/api/{page}", apiHandler).Methods("GET") + router.HandleFunc("/api/view/{id}", apiViewHandler).Methods("GET") router.HandleFunc("/faq", faqHandler).Name("faq") + router.HandleFunc("/feed.xml", rssHandler) + router.HandleFunc("/view/{id}", viewHandler).Name("view_torrent") + http.Handle("/", router) diff --git a/models.go b/models.go index 783ae1b2..57acdd79 100644 --- a/models.go +++ b/models.go @@ -9,6 +9,14 @@ import ( "strings" ) +type Feed struct { + Id int + Name string + Hash string + Magnet string + Timestamp string +} + type Categories struct { Id int `gorm:"column:category_id"` Name string `gorm:"column:category_name"` @@ -23,14 +31,24 @@ type Sub_Categories struct { Torrents []Torrents `gorm:"ForeignKey:sub_category_id;AssociationForeignKey:sub_category_id"` } +type Statuses struct { + Status_id int + Status_name string + Torrents []Torrents `gorm:"ForeignKey:status_id;AssociationForeignKey:status_id"` +} + type Torrents struct { - gorm.Model Id int `gorm:"column:torrent_id"` Name string `gorm:"column:torrent_name"` Category_id int `gorm:"column:category_id"` Sub_category_id int `gorm:"column:sub_category_id"` - Status int `gorm:"column:status_id"` + Status int `gorm:"column:status_id"` Hash string `gorm:"column:torrent_hash"` + Date int `gorm:"column:date"` + Downloads int `gorm:"column:downloads"` + Filesize string `gorm:"column:filesize"` + Description []byte `gorm:"column:description"` + Statuses Statuses `gorm:"ForeignKey:status_id;AssociationForeignKey:status_id"` Categories Categories `gorm:"ForeignKey:category_id;AssociationForeignKey:category_id"` Sub_Categories Sub_Categories `gorm:"ForeignKey:sub_category_id;AssociationForeignKey:sub_category_id"` } @@ -59,6 +77,9 @@ type TorrentsJson struct { Name string `json: "name"` Status int `json: "status"` Hash string `json: "hash"` + Date int `json: "date"` + Filesize string `json: "filesize"` + Description string `json: "description"` Sub_Category SubCategoryJson `json: "sub_category"` Category CategoryJson `json: "category"` Magnet template.URL `json: "magnet"` @@ -76,6 +97,27 @@ type WhereParams struct { * */ +// don't need raw SQL once we get MySQL +func getFeeds() []Feed { + var result []Feed + rows, err := db.DB(). + Query( + "SELECT `torrent_id` AS `id`, `torrent_name` AS `name`, `torrent_hash` AS `hash`, `timestamp` FROM `torrents` " + + "ORDER BY `timestamp` desc LIMIT 50") + if err == nil { + for rows.Next() { + item := Feed{} + rows.Scan(&item.Id, &item.Name, &item.Hash, &item.Timestamp) + magnet := "magnet:?xt=urn:btih:" + strings.TrimSpace(item.Hash) + "&dn=" + item.Name + trackers + item.Magnet = magnet + // memory hog + result = append(result, item) + } + rows.Close() + } + return result +} + func getTorrentById(id string) (Torrents, error) { var torrent Torrents @@ -96,18 +138,21 @@ func getTorrentsOrderBy(parameters *WhereParams, orderBy string, limit int, offs } else { db.Model(&torrents).Count(&count) dbQuery = db.Model(&torrents) - } - - if (orderBy == "") { orderBy = "torrent_id DESC" } // Default OrderBy + } + + if orderBy == "" { + orderBy = "torrent_id DESC" + } // Default OrderBy if limit != 0 || offset != 0 { // if limits provided dbQuery = dbQuery.Limit(limit).Offset(offset) } dbQuery.Order(orderBy).Preload("Categories").Preload("Sub_Categories").Find(&torrents) return torrents, count + } -/* Functions to simplify the get parameters of the main function - * +/* Functions to simplify the get parameters of the main function + * * Get Torrents with where parameters and limits, order by default */ func getTorrents(parameters WhereParams, limit int, offset int) ([]Torrents, int) { @@ -124,6 +169,7 @@ func getTorrentsDB(parameters WhereParams) ([]Torrents, int) { */ func getAllTorrentsOrderBy(orderBy string, limit int, offset int) ([]Torrents, int) { + return getTorrentsOrderBy(nil, orderBy, limit, offset) } @@ -166,9 +212,13 @@ func (t *Torrents) toJson() TorrentsJson { Name: html.UnescapeString(t.Name), Status: t.Status, Hash: t.Hash, + Date: t.Date, + Filesize: t.Filesize, + Description: unZlib(t.Description), Sub_Category: t.Sub_Categories.toJson(), Category: t.Categories.toJson(), Magnet: safe(magnet)} + return res } @@ -184,4 +234,4 @@ func (c *Categories) toJson() CategoryJson { Name: html.UnescapeString(c.Name)} } -/* Complete the functions when necessary... */ \ No newline at end of file +/* Complete the functions when necessary... */ diff --git a/package.sh b/package.sh new file mode 100755 index 00000000..973f219d --- /dev/null +++ b/package.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Helper script to ease building binary packages for multiple targets. +# Requires the linux64 and mingw64 gcc compilers and zip. +# On Debian-based distros install mingw-w64. + +version=$(git describe --tags) +declare -a OSes +OSes[0]='linux;x86_64-linux-gnu-gcc' +OSes[1]='windows;x86_64-w64-mingw32-gcc' + +for i in "${OSes[@]}"; do + arr=(${i//;/ }) + os=${arr[0]} + cc=${arr[1]} + rm -f nyaa nyaa.exe + echo -e "\nBuilding $os..." + echo GOOS=$os GOARCH=amd64 CC=$cc CGO_ENABLED=1 go build -v + GOOS=$os GOARCH=amd64 CC=$cc CGO_ENABLED=1 go build -v + zip -9 -q nyaa-${version}_${os}_amd64.zip css js *.md *.html nyaa nyaa.exe +done diff --git a/templateVariables.go b/templateVariables.go index b7e6951a..1951f9b1 100644 --- a/templateVariables.go +++ b/templateVariables.go @@ -15,6 +15,8 @@ type FaqTemplateVariables struct { Query string Status string Category string + Sort string + Order string Navigation Navigation } @@ -24,6 +26,8 @@ type HomeTemplateVariables struct { Query string Status string Category string + Sort string + Order string Navigation Navigation URL *url.URL // For parsing Url in templates Route *mux.Route // For getting current route in templates diff --git a/templates/home.html b/templates/home.html index 7783ebe9..bab2bf8d 100644 --- a/templates/home.html +++ b/templates/home.html @@ -6,6 +6,8 @@ Id Name Hash + Date + Size Links {{ range .ListTorrents}} @@ -13,10 +15,13 @@ {{if eq .Status 3}}class="trusted"{{end}} {{if eq .Status 4}}class="aplus"{{end}}> - {{.Name}} + {{.Name}} {{.Hash}} + {{.Date}} + {{.Filesize}} + diff --git a/templates/index.html b/templates/index.html index ef954ff9..51a2a8c3 100644 --- a/templates/index.html +++ b/templates/index.html @@ -35,10 +35,9 @@ + + + +
+
+

Advanced Search

+