Reduce number of queries, update systemd unit service (#925)

* Update/add systemd services

* Avoid roundtrip back to postgresql when doing ES search

* Use only one ES client

common/torrent.go

@@ -12,7 +12,6 @@ import (
 	elastic "gopkg.in/olivere/elastic.v5"
 
 	"github.com/NyaaPantsu/nyaa/config"
-	"github.com/NyaaPantsu/nyaa/db"
 	"github.com/NyaaPantsu/nyaa/model"
 	"github.com/NyaaPantsu/nyaa/util/log"
 )
@@ -171,9 +170,6 @@ func (p *TorrentParam) Find(client *elastic.Client) (int64, []model.Torrent, err
 			DefaultOperator("AND")
 	}
 
-	fsc := elastic.NewFetchSourceContext(true).
-		Include("id")
-
 	// TODO Find a better way to keep in sync with mapping in ansible
 	search := client.Search().
 		Index(config.Conf.Search.ElasticsearchIndex).
@@ -182,8 +178,7 @@ func (p *TorrentParam) Find(client *elastic.Client) (int64, []model.Torrent, err
 		From(int((p.Offset-1)*p.Max)).
 		Size(int(p.Max)).
 		Sort(p.Sort.ToESField(), p.Order).
-		Sort("_score", false). // Don't put _score before the field sort, it messes with the sorting
-		FetchSourceContext(fsc)
+		Sort("_score", false) // Don't put _score before the field sort, it messes with the sorting
 
 	filterQueryString := p.ToFilterQuery()
 	if filterQueryString != "" {
@@ -200,40 +195,21 @@ func (p *TorrentParam) Find(client *elastic.Client) (int64, []model.Torrent, err
 	log.Infof("Query '%s' took %d milliseconds.", p.NameLike, result.TookInMillis)
 	log.Infof("Amount of results %d.", result.TotalHits())
 
-	/* TODO Cleanup this giant mess
-	 * The raw query is used because we need to preserve the order of the id's
-	 * in the IN clause, so we can't just do
-	 *      select * from torrents where torrent_id IN (list_of_ids)
-	 * This query is said to work on postgres 9.4+
-	 */
-	{
-		// Temporary struct to hold the id
-		// INFO We are not using Hits.Id because the id in the index might not
-		// correspond to the id in the database later on.
-		type TId struct {
-			Id uint
-		}
-		var tid TId
-		var torrents []model.Torrent
-		if len(result.Hits.Hits) > 0 {
-			torrents = make([]model.Torrent, len(result.Hits.Hits))
-			hits := result.Hits.Hits
-			// Building a string of the form {id1,id2,id3}
-			source, _ := hits[0].Source.MarshalJSON()
-			json.Unmarshal(source, &tid)
-			idsToString := "{" + strconv.FormatUint(uint64(tid.Id), 10)
-			for _, t := range hits[1:] {
-				source, _ = t.Source.MarshalJSON()
-				json.Unmarshal(source, &tid)
-				idsToString += "," + strconv.FormatUint(uint64(tid.Id), 10)
-			}
-			idsToString += "}"
-			db.ORM.Raw("SELECT * FROM " + config.Conf.Models.TorrentsTableName +
-				" JOIN unnest('" + idsToString + "'::int[]) " +
-				" WITH ORDINALITY t(torrent_id, ord) USING (torrent_id) ORDER  BY t.ord").Find(&torrents)
-		}
-		return result.TotalHits(), torrents, nil
+	torrents := make([]model.Torrent, len(result.Hits.Hits))
+	if len(result.Hits.Hits) <= 0 {
+		return 0, nil, nil
 	}
+	for i, hit := range result.Hits.Hits {
+		// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
+		var tJson model.TorrentJSON
+		err := json.Unmarshal(*hit.Source, &tJson)
+		if err != nil {
+			log.Errorf("Cannot unmarshal elasticsearch torrent: %s", err)
+		}
+		torrent := tJson.ToTorrent()
+		torrents[i] = torrent
+	}
+	return result.TotalHits(), torrents, nil
 
 }
 

db/gorm.go

@@ -7,6 +7,7 @@ import (
 	"github.com/azhao12345/gorm"
 	_ "github.com/jinzhu/gorm/dialects/postgres" // Need for postgres support
 	_ "github.com/jinzhu/gorm/dialects/sqlite"   // Need for sqlite
+	elastic "gopkg.in/olivere/elastic.v5"
 )
 
 const (
@@ -23,10 +24,22 @@ var DefaultLogger Logger
 
 // ORM : Variable for interacting with database
 var ORM *gorm.DB
+var ElasticSearchClient *elastic.Client
 
 // IsSqlite : Variable to know if we are in sqlite or postgres
 var IsSqlite bool
 
+func ElasticSearchInit() (*elastic.Client, error) {
+	client, err := elastic.NewClient()
+	if err != nil {
+		log.Errorf("Unable to create elasticsearch client: %s", err)
+		return nil, err
+	} else {
+		log.Infof("Using elasticsearch client")
+		return client, nil
+	}
+}
+
 // GormInit init gorm ORM.
 func GormInit(conf *config.Config, logger Logger) (*gorm.DB, error) {
 

main.go

@@ -145,6 +145,7 @@ func main() {
 		if err != nil {
 			log.Fatal(err.Error())
 		}
+		db.ElasticSearchClient, _ = db.ElasticSearchInit()
 		err = publicSettings.InitI18n(conf.I18n, userService.NewCurrentUserRetriever())
 		if err != nil {
 			log.Fatal(err.Error())

model/torrent.go

@@ -228,6 +228,52 @@ type TorrentJSON struct {
 	FileList     []FileJSON    `json:"file_list"`
 }
 
+// TODO: Need to get rid of TorrentJSON altogether and have only one true Torrent
+//       model
+func (t *TorrentJSON) ToTorrent() Torrent {
+	category, err := strconv.ParseInt(t.Category, 10, 64)
+	if err != nil {
+		category = 0
+	}
+	subCategory, err := strconv.ParseInt(t.SubCategory, 10, 64)
+	if err != nil {
+		subCategory = 0
+	}
+	// Need to add +00:00 at the end because ES doesn't store it by default
+	date, err := time.Parse(time.RFC3339, t.Date + "+00:00")
+	if err != nil {
+		// TODO: Not sure what I should do here
+		date = time.Now()
+	}
+	torrent := Torrent{
+		ID: t.ID,
+		Name: t.Name,
+		Hash: t.Hash,
+		Category: int(category),
+		SubCategory: int(subCategory),
+		Status: t.Status,
+		Date: date,
+		UploaderID: t.UploaderID,
+		Downloads: t.Downloads,
+		//Stardom: t.Stardom,
+		Filesize: t.Filesize,
+		//Description: t.Description,
+		//WebsiteLink: t.WebsiteLink,
+		//Trackers: t.Trackers,
+		//DeletedAt: t.DeletedAt,
+		// Uploader: TODO
+		//OldUploader: t.OldUploader,
+		//OldComments: TODO
+		// Comments: TODO
+		Seeders: t.Seeders,
+		Leechers: t.Leechers,
+		Completed: t.Completed,
+		LastScrape: t.LastScrape,
+		//FileList: TODO
+	}
+	return torrent
+}
+
 // ToJSON converts a model.Torrent to its equivalent JSON structure
 func (t *Torrent) ToJSON() TorrentJSON {
 	var trackers []string

os/nyaa.service

@@ -1,10 +1,10 @@
 [Unit]
-Description=Torrent indexer for weebs
-After=network.target
+Description=Nyaa torrent
+After=network.target elasticsearch.service postgresql-9.6.service pgpool-II-96.service
 
 [Service]
-WorkingDirectory=/srv/nyaa/
-ExecStart=/usr/local/bin/nyaa -dbtype sqlite3 -dbparams ./nyaa.db
+WorkingDirectory=/home/nyaapantsu/go/src/github.com/NyaaPantsu/nyaa/
+ExecStart=/home/nyaapantsu/go/src/github.com/NyaaPantsu/nyaa/nyaa -conf=config/nyaa.yml
 StandardOutput=syslog
 Restart=on-failure
 

os/sukebei.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Sukebei torrent
+After=network.target elasticsearch.service postgresql-9.6.service pgpool-II-96.service
+
+[Service]
+WorkingDirectory=/home/nyaapantsu/go/src/github.com/NyaaPantsu/nyaa/
+ExecStart=/home/nyaapantsu/go/src/github.com/NyaaPantsu/nyaa/nyaa -conf=config/sukebei.yml
+StandardOutput=syslog
+Restart=on-failure
+
+[Install]
+WantedBy=default.target

router/api_handler.go

@@ -9,8 +9,6 @@ import (
 	"strings"
 	"time"
 
-	elastic "gopkg.in/olivere/elastic.v5"
-
 	"github.com/NyaaPantsu/nyaa/config"
 	"github.com/NyaaPantsu/nyaa/db"
 	"github.com/NyaaPantsu/nyaa/model"
@@ -218,16 +216,13 @@ func APIUploadHandler(w http.ResponseWriter, r *http.Request) {
 
 		db.ORM.Create(&torrent)
 
-		client, err := elastic.NewClient()
-		if err == nil {
-			err = torrent.AddToESIndex(client)
+		if db.ElasticSearchClient != nil {
+			err := torrent.AddToESIndex(db.ElasticSearchClient)
 			if err == nil {
 				log.Infof("Successfully added torrent to ES index.")
 			} else {
 				log.Errorf("Unable to add torrent to ES index: %s", err)
 			}
-		} else {
-			log.Errorf("Unable to create elasticsearch client: %s", err)
 		}
 		/*if err != nil {
 			util.SendError(w, err, 500)

router/upload_handler.go

@@ -6,8 +6,6 @@ import (
 	"strconv"
 	"time"
 
-	elastic "gopkg.in/olivere/elastic.v5"
-
 	"github.com/NyaaPantsu/nyaa/config"
 	"github.com/NyaaPantsu/nyaa/db"
 	"github.com/NyaaPantsu/nyaa/model"
@@ -91,16 +89,13 @@ func UploadPostHandler(w http.ResponseWriter, r *http.Request) {
 		torrent.ParseTrackers(uploadForm.Trackers)
 		db.ORM.Create(&torrent)
 
-		client, err := elastic.NewClient()
-		if err == nil {
-			err = torrent.AddToESIndex(client)
+		if db.ElasticSearchClient != nil {
+			err := torrent.AddToESIndex(db.ElasticSearchClient)
 			if err == nil {
 				log.Infof("Successfully added torrent to ES index.")
 			} else {
 				log.Errorf("Unable to add torrent to ES index: %s", err)
 			}
-		} else {
-			log.Errorf("Unable to create elasticsearch client: %s", err)
 		}
 
 		url, err := Router.Get("view_torrent").URL("id", strconv.FormatUint(uint64(torrent.ID), 10))

service/torrent/torrent.go

@@ -6,8 +6,6 @@ import (
 	"strconv"
 	"strings"
 
-	elastic "gopkg.in/olivere/elastic.v5"
-
 	"github.com/NyaaPantsu/nyaa/config"
 	"github.com/NyaaPantsu/nyaa/db"
 	"github.com/NyaaPantsu/nyaa/model"
@@ -225,17 +223,13 @@ func DeleteTorrent(id string) (int, error) {
 		return http.StatusInternalServerError, errors.New("Torrent was not deleted")
 	}
 
-	// TODO Don't create a new client for each request
-	client, err := elastic.NewClient()
-	if err == nil {
-		err = torrent.DeleteFromESIndex(client)
+	if db.ElasticSearchClient != nil {
+		err := torrent.DeleteFromESIndex(db.ElasticSearchClient)
 		if err == nil {
 			log.Infof("Successfully deleted torrent to ES index.")
 		} else {
 			log.Errorf("Unable to delete torrent to ES index: %s", err)
 		}
-	} else {
-		log.Errorf("Unable to create elasticsearch client: %s", err)
 	}
 	return http.StatusOK, nil
 }
@@ -250,17 +244,13 @@ func DefinitelyDeleteTorrent(id string) (int, error) {
 		return http.StatusInternalServerError, errors.New("Torrent was not deleted")
 	}
 
-	// TODO Don't create a new client for each request
-	client, err := elastic.NewClient()
-	if err == nil {
-		err = torrent.DeleteFromESIndex(client)
+	if db.ElasticSearchClient != nil {
+		err := torrent.DeleteFromESIndex(db.ElasticSearchClient)
 		if err == nil {
 			log.Infof("Successfully deleted torrent to ES index.")
 		} else {
 			log.Errorf("Unable to delete torrent to ES index: %s", err)
 		}
-	} else {
-		log.Errorf("Unable to create elasticsearch client: %s", err)
 	}
 	return http.StatusOK, nil
 }
@@ -288,17 +278,14 @@ func UpdateTorrent(torrent model.Torrent) (int, error) {
 		return http.StatusInternalServerError, errors.New("Torrent was not updated")
 	}
 
-	// TODO Don't create a new client for each request
-	client, err := elastic.NewClient()
-	if err == nil {
-		err = torrent.AddToESIndex(client)
+// TODO Don't create a new client for each request
+	if db.ElasticSearchClient != nil {
+		err := torrent.AddToESIndex(db.ElasticSearchClient)
 		if err == nil {
 			log.Infof("Successfully updated torrent to ES index.")
 		} else {
 			log.Errorf("Unable to update torrent to ES index: %s", err)
 		}
-	} else {
-		log.Errorf("Unable to create elasticsearch client: %s", err)
 	}
 
 	return http.StatusOK, nil

util/search/search.go

@@ -8,8 +8,6 @@ import (
 	"unicode"
 	"unicode/utf8"
 
-	elastic "gopkg.in/olivere/elastic.v5"
-
 	"github.com/NyaaPantsu/nyaa/cache"
 	"github.com/NyaaPantsu/nyaa/common"
 	"github.com/NyaaPantsu/nyaa/config"
@@ -79,11 +77,10 @@ func SearchByQueryDeleted(r *http.Request, pagenum int) (search common.SearchPar
 func searchByQuery(r *http.Request, pagenum int, countAll bool, withUser bool, deleted bool) (
 	search common.SearchParam, tor []model.Torrent, count int, err error,
 ) {
-	client, err := elastic.NewClient()
-	if err == nil {
+	if db.ElasticSearchClient != nil {
 		var torrentParam common.TorrentParam
 		torrentParam.FromRequest(r)
-		totalHits, torrents, err := torrentParam.Find(client)
+		totalHits, torrents, err := torrentParam.Find(db.ElasticSearchClient)
 		searchParam := common.SearchParam{
 			TorrentID: uint(torrentParam.TorrentID),
 			FromID:    uint(torrentParam.FromID),