From e60eceba6340aaffab81655d311588353939839b Mon Sep 17 00:00:00 2001 From: tomleb <tomleb@users.noreply.github.com> Date: Sun, 4 Jun 2017 21:33:02 -0400 Subject: [PATCH] Reduce number of queries, update systemd unit service (#925) * Update/add systemd services * Avoid roundtrip back to postgresql when doing ES search * Use only one ES client --- common/torrent.go | 54 +++++++++++--------------------------- db/gorm.go | 13 +++++++++ main.go | 1 + model/torrent.go | 46 ++++++++++++++++++++++++++++++++ os/nyaa.service | 8 +++--- os/sukebei.service | 12 +++++++++ router/api_handler.go | 9 ++----- router/upload_handler.go | 9 ++----- service/torrent/torrent.go | 27 +++++-------------- util/search/search.go | 7 ++--- 10 files changed, 104 insertions(+), 82 deletions(-) create mode 100644 os/sukebei.service diff --git a/common/torrent.go b/common/torrent.go index 99216a1a..103cb7c8 100644 --- a/common/torrent.go +++ b/common/torrent.go @@ -12,7 +12,6 @@ import ( elastic "gopkg.in/olivere/elastic.v5" "github.com/NyaaPantsu/nyaa/config" - "github.com/NyaaPantsu/nyaa/db" "github.com/NyaaPantsu/nyaa/model" "github.com/NyaaPantsu/nyaa/util/log" ) @@ -171,9 +170,6 @@ func (p *TorrentParam) Find(client *elastic.Client) (int64, []model.Torrent, err DefaultOperator("AND") } - fsc := elastic.NewFetchSourceContext(true). - Include("id") - // TODO Find a better way to keep in sync with mapping in ansible search := client.Search(). Index(config.Conf.Search.ElasticsearchIndex). @@ -182,8 +178,7 @@ func (p *TorrentParam) Find(client *elastic.Client) (int64, []model.Torrent, err From(int((p.Offset-1)*p.Max)). Size(int(p.Max)). Sort(p.Sort.ToESField(), p.Order). - Sort("_score", false). // Don't put _score before the field sort, it messes with the sorting - FetchSourceContext(fsc) + Sort("_score", false) // Don't put _score before the field sort, it messes with the sorting filterQueryString := p.ToFilterQuery() if filterQueryString != "" { @@ -200,40 +195,21 @@ func (p *TorrentParam) Find(client *elastic.Client) (int64, []model.Torrent, err log.Infof("Query '%s' took %d milliseconds.", p.NameLike, result.TookInMillis) log.Infof("Amount of results %d.", result.TotalHits()) - /* TODO Cleanup this giant mess - * The raw query is used because we need to preserve the order of the id's - * in the IN clause, so we can't just do - * select * from torrents where torrent_id IN (list_of_ids) - * This query is said to work on postgres 9.4+ - */ - { - // Temporary struct to hold the id - // INFO We are not using Hits.Id because the id in the index might not - // correspond to the id in the database later on. - type TId struct { - Id uint - } - var tid TId - var torrents []model.Torrent - if len(result.Hits.Hits) > 0 { - torrents = make([]model.Torrent, len(result.Hits.Hits)) - hits := result.Hits.Hits - // Building a string of the form {id1,id2,id3} - source, _ := hits[0].Source.MarshalJSON() - json.Unmarshal(source, &tid) - idsToString := "{" + strconv.FormatUint(uint64(tid.Id), 10) - for _, t := range hits[1:] { - source, _ = t.Source.MarshalJSON() - json.Unmarshal(source, &tid) - idsToString += "," + strconv.FormatUint(uint64(tid.Id), 10) - } - idsToString += "}" - db.ORM.Raw("SELECT * FROM " + config.Conf.Models.TorrentsTableName + - " JOIN unnest('" + idsToString + "'::int[]) " + - " WITH ORDINALITY t(torrent_id, ord) USING (torrent_id) ORDER BY t.ord").Find(&torrents) - } - return result.TotalHits(), torrents, nil + torrents := make([]model.Torrent, len(result.Hits.Hits)) + if len(result.Hits.Hits) <= 0 { + return 0, nil, nil } + for i, hit := range result.Hits.Hits { + // Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}). + var tJson model.TorrentJSON + err := json.Unmarshal(*hit.Source, &tJson) + if err != nil { + log.Errorf("Cannot unmarshal elasticsearch torrent: %s", err) + } + torrent := tJson.ToTorrent() + torrents[i] = torrent + } + return result.TotalHits(), torrents, nil } diff --git a/db/gorm.go b/db/gorm.go index 16e96130..fca134f5 100644 --- a/db/gorm.go +++ b/db/gorm.go @@ -7,6 +7,7 @@ import ( "github.com/azhao12345/gorm" _ "github.com/jinzhu/gorm/dialects/postgres" // Need for postgres support _ "github.com/jinzhu/gorm/dialects/sqlite" // Need for sqlite + elastic "gopkg.in/olivere/elastic.v5" ) const ( @@ -23,10 +24,22 @@ var DefaultLogger Logger // ORM : Variable for interacting with database var ORM *gorm.DB +var ElasticSearchClient *elastic.Client // IsSqlite : Variable to know if we are in sqlite or postgres var IsSqlite bool +func ElasticSearchInit() (*elastic.Client, error) { + client, err := elastic.NewClient() + if err != nil { + log.Errorf("Unable to create elasticsearch client: %s", err) + return nil, err + } else { + log.Infof("Using elasticsearch client") + return client, nil + } +} + // GormInit init gorm ORM. func GormInit(conf *config.Config, logger Logger) (*gorm.DB, error) { diff --git a/main.go b/main.go index 7621769f..9dae0f05 100644 --- a/main.go +++ b/main.go @@ -145,6 +145,7 @@ func main() { if err != nil { log.Fatal(err.Error()) } + db.ElasticSearchClient, _ = db.ElasticSearchInit() err = publicSettings.InitI18n(conf.I18n, userService.NewCurrentUserRetriever()) if err != nil { log.Fatal(err.Error()) diff --git a/model/torrent.go b/model/torrent.go index 760ec9a3..e02b07d3 100644 --- a/model/torrent.go +++ b/model/torrent.go @@ -228,6 +228,52 @@ type TorrentJSON struct { FileList []FileJSON `json:"file_list"` } +// TODO: Need to get rid of TorrentJSON altogether and have only one true Torrent +// model +func (t *TorrentJSON) ToTorrent() Torrent { + category, err := strconv.ParseInt(t.Category, 10, 64) + if err != nil { + category = 0 + } + subCategory, err := strconv.ParseInt(t.SubCategory, 10, 64) + if err != nil { + subCategory = 0 + } + // Need to add +00:00 at the end because ES doesn't store it by default + date, err := time.Parse(time.RFC3339, t.Date + "+00:00") + if err != nil { + // TODO: Not sure what I should do here + date = time.Now() + } + torrent := Torrent{ + ID: t.ID, + Name: t.Name, + Hash: t.Hash, + Category: int(category), + SubCategory: int(subCategory), + Status: t.Status, + Date: date, + UploaderID: t.UploaderID, + Downloads: t.Downloads, + //Stardom: t.Stardom, + Filesize: t.Filesize, + //Description: t.Description, + //WebsiteLink: t.WebsiteLink, + //Trackers: t.Trackers, + //DeletedAt: t.DeletedAt, + // Uploader: TODO + //OldUploader: t.OldUploader, + //OldComments: TODO + // Comments: TODO + Seeders: t.Seeders, + Leechers: t.Leechers, + Completed: t.Completed, + LastScrape: t.LastScrape, + //FileList: TODO + } + return torrent +} + // ToJSON converts a model.Torrent to its equivalent JSON structure func (t *Torrent) ToJSON() TorrentJSON { var trackers []string diff --git a/os/nyaa.service b/os/nyaa.service index fdac684f..542ee609 100644 --- a/os/nyaa.service +++ b/os/nyaa.service @@ -1,10 +1,10 @@ [Unit] -Description=Torrent indexer for weebs -After=network.target +Description=Nyaa torrent +After=network.target elasticsearch.service postgresql-9.6.service pgpool-II-96.service [Service] -WorkingDirectory=/srv/nyaa/ -ExecStart=/usr/local/bin/nyaa -dbtype sqlite3 -dbparams ./nyaa.db +WorkingDirectory=/home/nyaapantsu/go/src/github.com/NyaaPantsu/nyaa/ +ExecStart=/home/nyaapantsu/go/src/github.com/NyaaPantsu/nyaa/nyaa -conf=config/nyaa.yml StandardOutput=syslog Restart=on-failure diff --git a/os/sukebei.service b/os/sukebei.service new file mode 100644 index 00000000..3e84beef --- /dev/null +++ b/os/sukebei.service @@ -0,0 +1,12 @@ +[Unit] +Description=Sukebei torrent +After=network.target elasticsearch.service postgresql-9.6.service pgpool-II-96.service + +[Service] +WorkingDirectory=/home/nyaapantsu/go/src/github.com/NyaaPantsu/nyaa/ +ExecStart=/home/nyaapantsu/go/src/github.com/NyaaPantsu/nyaa/nyaa -conf=config/sukebei.yml +StandardOutput=syslog +Restart=on-failure + +[Install] +WantedBy=default.target diff --git a/router/api_handler.go b/router/api_handler.go index 3f25e10b..7cce318a 100644 --- a/router/api_handler.go +++ b/router/api_handler.go @@ -9,8 +9,6 @@ import ( "strings" "time" - elastic "gopkg.in/olivere/elastic.v5" - "github.com/NyaaPantsu/nyaa/config" "github.com/NyaaPantsu/nyaa/db" "github.com/NyaaPantsu/nyaa/model" @@ -218,16 +216,13 @@ func APIUploadHandler(w http.ResponseWriter, r *http.Request) { db.ORM.Create(&torrent) - client, err := elastic.NewClient() - if err == nil { - err = torrent.AddToESIndex(client) + if db.ElasticSearchClient != nil { + err := torrent.AddToESIndex(db.ElasticSearchClient) if err == nil { log.Infof("Successfully added torrent to ES index.") } else { log.Errorf("Unable to add torrent to ES index: %s", err) } - } else { - log.Errorf("Unable to create elasticsearch client: %s", err) } /*if err != nil { util.SendError(w, err, 500) diff --git a/router/upload_handler.go b/router/upload_handler.go index ba03a175..1e3f91ae 100644 --- a/router/upload_handler.go +++ b/router/upload_handler.go @@ -6,8 +6,6 @@ import ( "strconv" "time" - elastic "gopkg.in/olivere/elastic.v5" - "github.com/NyaaPantsu/nyaa/config" "github.com/NyaaPantsu/nyaa/db" "github.com/NyaaPantsu/nyaa/model" @@ -91,16 +89,13 @@ func UploadPostHandler(w http.ResponseWriter, r *http.Request) { torrent.ParseTrackers(uploadForm.Trackers) db.ORM.Create(&torrent) - client, err := elastic.NewClient() - if err == nil { - err = torrent.AddToESIndex(client) + if db.ElasticSearchClient != nil { + err := torrent.AddToESIndex(db.ElasticSearchClient) if err == nil { log.Infof("Successfully added torrent to ES index.") } else { log.Errorf("Unable to add torrent to ES index: %s", err) } - } else { - log.Errorf("Unable to create elasticsearch client: %s", err) } url, err := Router.Get("view_torrent").URL("id", strconv.FormatUint(uint64(torrent.ID), 10)) diff --git a/service/torrent/torrent.go b/service/torrent/torrent.go index 48d96dc5..25fb193d 100644 --- a/service/torrent/torrent.go +++ b/service/torrent/torrent.go @@ -6,8 +6,6 @@ import ( "strconv" "strings" - elastic "gopkg.in/olivere/elastic.v5" - "github.com/NyaaPantsu/nyaa/config" "github.com/NyaaPantsu/nyaa/db" "github.com/NyaaPantsu/nyaa/model" @@ -225,17 +223,13 @@ func DeleteTorrent(id string) (int, error) { return http.StatusInternalServerError, errors.New("Torrent was not deleted") } - // TODO Don't create a new client for each request - client, err := elastic.NewClient() - if err == nil { - err = torrent.DeleteFromESIndex(client) + if db.ElasticSearchClient != nil { + err := torrent.DeleteFromESIndex(db.ElasticSearchClient) if err == nil { log.Infof("Successfully deleted torrent to ES index.") } else { log.Errorf("Unable to delete torrent to ES index: %s", err) } - } else { - log.Errorf("Unable to create elasticsearch client: %s", err) } return http.StatusOK, nil } @@ -250,17 +244,13 @@ func DefinitelyDeleteTorrent(id string) (int, error) { return http.StatusInternalServerError, errors.New("Torrent was not deleted") } - // TODO Don't create a new client for each request - client, err := elastic.NewClient() - if err == nil { - err = torrent.DeleteFromESIndex(client) + if db.ElasticSearchClient != nil { + err := torrent.DeleteFromESIndex(db.ElasticSearchClient) if err == nil { log.Infof("Successfully deleted torrent to ES index.") } else { log.Errorf("Unable to delete torrent to ES index: %s", err) } - } else { - log.Errorf("Unable to create elasticsearch client: %s", err) } return http.StatusOK, nil } @@ -288,17 +278,14 @@ func UpdateTorrent(torrent model.Torrent) (int, error) { return http.StatusInternalServerError, errors.New("Torrent was not updated") } - // TODO Don't create a new client for each request - client, err := elastic.NewClient() - if err == nil { - err = torrent.AddToESIndex(client) +// TODO Don't create a new client for each request + if db.ElasticSearchClient != nil { + err := torrent.AddToESIndex(db.ElasticSearchClient) if err == nil { log.Infof("Successfully updated torrent to ES index.") } else { log.Errorf("Unable to update torrent to ES index: %s", err) } - } else { - log.Errorf("Unable to create elasticsearch client: %s", err) } return http.StatusOK, nil diff --git a/util/search/search.go b/util/search/search.go index af6d8997..3af73b7b 100644 --- a/util/search/search.go +++ b/util/search/search.go @@ -8,8 +8,6 @@ import ( "unicode" "unicode/utf8" - elastic "gopkg.in/olivere/elastic.v5" - "github.com/NyaaPantsu/nyaa/cache" "github.com/NyaaPantsu/nyaa/common" "github.com/NyaaPantsu/nyaa/config" @@ -79,11 +77,10 @@ func SearchByQueryDeleted(r *http.Request, pagenum int) (search common.SearchPar func searchByQuery(r *http.Request, pagenum int, countAll bool, withUser bool, deleted bool) ( search common.SearchParam, tor []model.Torrent, count int, err error, ) { - client, err := elastic.NewClient() - if err == nil { + if db.ElasticSearchClient != nil { var torrentParam common.TorrentParam torrentParam.FromRequest(r) - totalHits, torrents, err := torrentParam.Find(client) + totalHits, torrents, err := torrentParam.Find(db.ElasticSearchClient) searchParam := common.SearchParam{ TorrentID: uint(torrentParam.TorrentID), FromID: uint(torrentParam.FromID),