Merge pull request #84 from tomleb/deployment

Adding Docker support (WIP)

.gitignore

@@ -1,5 +1,6 @@
 *.sqlite
 *.db
+*.sql
 main
 nyaa
 nyaa.exe
@@ -8,3 +9,5 @@ nyaa-master.exe
 *.swp
 .vscode
 templates/*.html.go
+*.backup
+tags

.travis.yml

@@ -6,6 +6,7 @@ install:
 - go get github.com/gorilla/mux
 - go get github.com/mattn/go-sqlite3
 - go get github.com/jinzhu/gorm
+- go get github.com/lib/pq
 - go get github.com/Sirupsen/logrus
 - go get gopkg.in/natefinch/lumberjack.v2
 - go get gopkg.in/gomail.v2

README.md

@@ -35,6 +35,30 @@ The provided unit file uses options directly; if you prefer a config file, do th
 * Edit `nyaa.conf` to your liking
 * Replace in the unit file the options by `-conf /etc/nyaa.conf`
 
+
+## Docker
+
+We support docker for easy development and deployment. Simply install docker and
+docker-compose by following the instructions [here](https://docs.docker.com/engine/installation/linux/ubuntu/#install-using-the-repository).
+
+Once you've successfully installed docker, make sure you have the database file
+in the project's directory as nyaa.db. Then, follow these steps to build and run
+the application.
+
+```sh
+# Make sure the project is in here $GOPATH/src/github.com/ewhal/nyaa
+$ cd deploy/
+# You may choose another backend by pointing to the
+# appropriate docker-compose file.
+$ docker-compose -f docker-compose.sqlite.yml build
+$ docker-compose -f docker-compose.sqlite.yml up 
+```
+
+Access the website by going to [localhost:9999](http://localhost:9999).
+
+> For postgres, place the dump in the toplevel directory and name it to
+> nyaa_psql.backup.
+
 ## TODO
 
 ### Features until stable release

db/gorm.go

@@ -6,6 +6,7 @@ import (
 	"github.com/ewhal/nyaa/util/log"
 	"github.com/jinzhu/gorm"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 	// _ "github.com/go-sql-driver/mysql"
 )
 

deploy/.env

@@ -0,0 +1,3 @@
+PANTSU_EXTERNAL_PORT=9999
+PANTSU_INTERNAL_PORT=9999
+PANTSU_POSTGRES_DBFILE=nyaa_psql.backup

deploy/Dockerfile

@@ -0,0 +1,5 @@
+FROM golang:1.8.1
+
+RUN mkdir -p /nyaa
+
+WORKDIR /nyaa

deploy/docker-compose.postgres-prod.yml

@@ -0,0 +1,19 @@
+version: '3'
+services:
+  pantsu:
+    build:
+      context: ..
+      dockerfile: deploy/Dockerfile
+    command: ./deploy/init.sh
+    env_file:
+      - postgres-prod.env
+    environment:
+      - PANTSU_INTERNAL_PORT=${PANTSU_INTERNAL_PORT}
+    ports:
+      # 0.0.0.0 makes it accessible to the network
+      # You may want to remove it to make pantsu available only
+      # to localhost
+      - 0.0.0.0:${PANTSU_EXTERNAL_PORT}:${PANTSU_INTERNAL_PORT}
+    volumes:
+      - "${GOPATH}:/go/"
+      - "./..:/nyaa"

deploy/docker-compose.postgres.yml

@@ -0,0 +1,34 @@
+version: '3'
+services:
+  pantsu:
+    build:
+      context: ..
+      dockerfile: deploy/Dockerfile
+    command: ./deploy/init.sh
+    depends_on: 
+      - pantsu_db
+    env_file:
+      - postgres.env
+    environment:
+      - PANTSU_INTERNAL_PORT=${PANTSU_INTERNAL_PORT}
+    links:
+      - pantsu_db
+    ports:
+      # 0.0.0.0 makes it accessible to the network
+      # You may want to remove it to make pantsu available only
+      # to localhost
+      - 0.0.0.0:${PANTSU_EXTERNAL_PORT}:${PANTSU_INTERNAL_PORT}
+    volumes:
+      - "${GOPATH}:/go/"
+      - "./..:/nyaa"
+
+  pantsu_db:
+    env_file:
+      - postgres.env
+    environment:
+      - PANTSU_POSTGRES_DBFILE=${PANTSU_POSTGRES_DBFILE}
+    image: postgres:9.6.2
+    volumes:
+      # restore.sh will be sourced after initial initdb
+      - "./restore.sh:/docker-entrypoint-initdb.d/restore.sh"
+      - "./../${PANTSU_POSTGRES_DBFILE}:/${PANTSU_POSTGRES_DBFILE}"

deploy/docker-compose.sqlite.yml

@@ -0,0 +1,19 @@
+version: '3'
+services:
+  pantsu:
+    build:
+      context: ..
+      dockerfile: deploy/Dockerfile
+    command: ./deploy/init.sh
+    env_file:
+      - sqlite.env
+    environment:
+      - PANTSU_INTERNAL_PORT=${PANTSU_INTERNAL_PORT}
+    ports:
+      # 0.0.0.0 makes it accessible to the network
+      # You may want to remove it to make pantsu available only
+      # to localhost
+      - 0.0.0.0:${PANTSU_EXTERNAL_PORT}:${PANTSU_INTERNAL_PORT}
+    volumes:
+      - "${GOPATH}:/go/"
+      - "./..:/nyaa"

deploy/init.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eux
+
+# TODO Doesn't scale, find another way to wait until db is ready
+if [[ "${PANTSU_DBTYPE}" = "postgres" ]]; then
+  echo 'Waiting for the database to be ready...'
+  sleep 40
+fi
+
+go get github.com/ewhal/nyaa
+go build
+./nyaa -host 0.0.0.0 \
+       -port "${PANTSU_INTERNAL_PORT}" \
+       -dbtype "${PANTSU_DBTYPE}" \
+       -dbparams "${PANTSU_DBPARAMS}"

deploy/postgres-prod.env

@@ -0,0 +1,9 @@
+POSTGRES_USER=nyaapantsu
+POSTGRES_PASSWORD=nyaapantsu
+POSTGRES_DB=nyaapantsu
+
+PANTSU_DBTYPE=postgres
+# TODO Would prefer to use the line below but docker doesn't seem to accept
+# that.
+#PANTSU_DBPARAMS=host=pantsu_db user=${POSTGRES_USER} dbname=${POSTGRES_DB} sslmode=disable password=${POSTGRES_PASSWORD}
+PANTSU_DBPARAMS=host=localhost user=nyaapantsu dbname=nyaapantsu sslmode=disable password=nyaapantsu

deploy/postgres.env

@@ -0,0 +1,9 @@
+POSTGRES_USER=nyaapantsu
+POSTGRES_PASSWORD=nyaapantsu
+POSTGRES_DB=nyaapantsu
+
+PANTSU_DBTYPE=postgres
+# TODO Would prefer to use the line below but docker doesn't seem to accept
+# that.
+#PANTSU_DBPARAMS=host=pantsu_db user=${POSTGRES_USER} dbname=${POSTGRES_DB} sslmode=disable password=${POSTGRES_PASSWORD}
+PANTSU_DBPARAMS=host=pantsu_db user=nyaapantsu dbname=nyaapantsu sslmode=disable password=nyaapantsu

deploy/prune_docker.sh

@@ -0,0 +1,11 @@
+# Prune images and volumes
+# 
+# Docker tends to take a lot of space. This will remove dangling images and
+# volumes not used by at least container.
+# WARNING: You might not want to run this if you have stuff that are dangling
+# that you want to keep.
+#
+#!/bin/bash
+
+docker image prune -f
+docker volume prune -f

deploy/restore.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Restore the database from a postgres dump
+
+pg_restore --username "${POSTGRES_USER}" -d ${POSTGRES_DB} "/${PANTSU_POSTGRES_DBFILE}"

deploy/sqlite.env

@@ -0,0 +1,2 @@
+PANTSU_DBTYPE=sqlite3
+PANTSU_DBPARAMS=./nyaa.db?cache_size=50

model/torrent.go

@@ -75,7 +75,7 @@ type TorrentsJson struct {
 func (t *Torrents) ToJson() TorrentsJson {
 	magnet := "magnet:?xt=urn:btih:" + strings.TrimSpace(t.Hash) + "&dn=" + t.Name + config.Trackers
 	b := []CommentsJson{}
-	_ = json.Unmarshal([]byte(util.UnZlib(t.Comments)), &b)
+	_ = json.Unmarshal([]byte(t.Comments), &b)
 	res := TorrentsJson{
 		Id:           strconv.Itoa(t.Id),
 		Name:         html.UnescapeString(t.Name),
@@ -83,7 +83,7 @@ func (t *Torrents) ToJson() TorrentsJson {
 		Hash:         t.Hash,
 		Date:         time.Unix(t.Date, 0).Format(time.RFC3339),
 		Filesize:     util.FormatFilesize(t.Filesize),
-		Description:  template.HTML(util.UnZlib(t.Description)),
+		Description:  template.HTML(t.Description),
 		Comments:     b,
 		Sub_Category: strconv.Itoa(t.Sub_Category),
 		Category:     strconv.Itoa(t.Category),

router/homeHandler.go

@@ -1,14 +1,13 @@
 package router
 
 import (
+	"github.com/ewhal/nyaa/model"
+	"github.com/ewhal/nyaa/service/torrent"
+	"github.com/gorilla/mux"
 	"html"
 	"html/template"
 	"net/http"
 	"strconv"
-
-	"github.com/ewhal/nyaa/model"
-	"github.com/ewhal/nyaa/service/torrent"
-	"github.com/gorilla/mux"
 )
 
 var homeTemplate = template.Must(template.New("home").Funcs(FuncMap).ParseFiles("templates/index.html", "templates/home.html"))

router/searchHandler.go

@@ -5,7 +5,6 @@ import (
 	"html/template"
 	"net/http"
 	"strconv"
-
 	"github.com/ewhal/nyaa/model"
 	"github.com/ewhal/nyaa/util/search"
 	"github.com/gorilla/mux"

service/torrent/torrent.go

@@ -71,7 +71,6 @@ func GetTorrentsOrderBy(parameters *WhereParams, orderBy string, limit int, offs
 	}
 	dbQuery.Order(orderBy).Find(&torrents)
 	return torrents, count
-
 }
 
 /* Functions to simplify the get parameters of the main function