add new db interface beginnings

common/comment.go

@@ -0,0 +1,8 @@
+package common
+
+type CommentParam struct {
+	UserID    uint32
+	CommentID uint32
+	Limit     uint32
+	Offset    uint32
+}

common/errors.go

@@ -0,0 +1,17 @@
+package common
+
+import (
+	"errors"
+)
+
+var ErrInsufficientPermission = errors.New("you are not allowed to do that")
+var ErrUserExists = errors.New("user already exists")
+var ErrNoSuchUser = errors.New("no such user")
+var ErrBadLogin = errors.New("bad login")
+var ErrUserBanned = errors.New("banned")
+var ErrBadEmail = errors.New("bad email")
+var ErrNoSuchEntry = errors.New("no such entry")
+var ErrNoSuchComment = errors.New("no such comment")
+var ErrNotFollowing = errors.New("not following that user")
+var ErrInvalidToken = errors.New("invalid token")
+var ErrExpiredToken = errors.New("token is expired")

common/report.go

@@ -0,0 +1,15 @@
+package common
+
+import (
+	"time"
+)
+
+type ReportParam struct {
+	Limit     uint32
+	Offset    uint32
+	AllTime   bool
+	ID        uint32
+	TorrentID uint32
+	Before    time.Time
+	After     time.Time
+}

common/scrape.go

@@ -0,0 +1,14 @@
+package common
+
+import (
+	"time"
+)
+
+type ScrapeResult struct {
+	Hash      string
+	TorrentID uint32
+	Seeders   uint32
+	Leechers  uint32
+	Completed uint32
+	Date      time.Time
+}

common/search.go

@@ -1,6 +1,10 @@
 package common
 
-import "strconv"
+import (
+	"strconv"
+	"strings"
+)
+
 
 type Status uint8
 
@@ -11,6 +15,22 @@ const (
 	APlus
 )
 
+func (st *Status) Parse(s string) {
+	switch s {
+	case "1":
+		*st = FilterRemakes
+		break
+	case "2":
+		*st = Trusted
+		break
+	case "3":
+		*st = APlus
+		break
+	default:
+		*st = ShowAll
+	}
+}
+
 type SortMode uint8
 
 const (
@@ -24,6 +44,34 @@ const (
 	Completed
 )
 
+func (s *SortMode) Parse(str string) {
+	switch str {
+	case "1":
+		*s = Name
+		break
+	case "2":
+		*s = Date
+		break
+	case "3":
+		*s = Downloads
+		break
+	case "4":
+		*s = Size
+		break
+	case "5":
+		*s = Seeders
+		break
+	case "6":
+		*s = Leechers
+		break
+	case "7":
+		*s = Completed
+		break
+	default:
+		*s = ID
+	}
+}
+
 type Category struct {
 	Main, Sub uint8
 }
@@ -39,14 +87,38 @@ func (c Category) String() (s string) {
 	return
 }
 
-type SearchParam struct {
-	Order    bool // True means acsending
-	Status   Status
-	Sort     SortMode
-	Category Category
-	Page     int
-	UserID   uint
-	Max      uint
-	NotNull  string
-	Query    string
+func (c Category) IsSet() bool {
+	return c.Main != 0 && c.Sub != 0
+}
+
+// Parse sets category by string
+// returns true if string is valid otherwise returns false
+func (c *Category) Parse(s string) (ok bool) {
+	parts := strings.Split(s, "_")
+	if len(parts) == 2 {
+		tmp, err := strconv.ParseUint(parts[0], 10, 8)
+		if err == nil {
+			c.Main = uint8(tmp)
+			tmp, err = strconv.ParseUint(parts[1], 10, 8)
+			if err == nil {
+				c.Sub = uint8(tmp)
+				ok = true
+			}
+		}
+	}
+	return
+}
+
+
+// deprecated for TorrentParam
+type SearchParam struct {
+       Order    bool // True means acsending
+       Status   Status
+       Sort     SortMode
+       Category Category
+       Page     int
+       UserID   uint
+       Max      uint
+       NotNull  string
+       Query    string
 }

common/torrent.go

@@ -0,0 +1,35 @@
+package common
+
+// TorrentParam defines all parameters that can be provided when searching for a torrent
+type TorrentParam struct {
+	All       bool // True means ignore everything but Max and Offset
+	Full      bool // True means load all members
+	Order     bool // True means acsending
+	Status    Status
+	Sort      SortMode
+	Category  Category
+	Max       uint32
+	Offset    uint32
+	UserID    uint32
+	TorrentID uint32
+	NotNull   string // csv
+	Null      string // csv
+	NameLike  string // csv
+}
+
+func (p *TorrentParam) Clone() TorrentParam {
+	return TorrentParam{
+		Order:     p.Order,
+		Status:    p.Status,
+		Sort:      p.Sort,
+		Category:  p.Category,
+		Max:       p.Max,
+		Offset:    p.Offset,
+		UserID:    p.UserID,
+		TorrentID: p.TorrentID,
+		NotNull:   p.NotNull,
+		Null:      p.Null,
+		NameLike:  p.NameLike,
+	}
+}
+

common/user.go

@@ -0,0 +1,11 @@
+package common
+
+type UserParam struct {
+	Full     bool // if true populate Uploads, UsersWeLiked and UsersLikingMe
+	Email    string
+	Name     string
+	ApiToken string
+	ID       uint32
+	Max      uint32
+	Offset   uint32
+}

database/database.go

@@ -0,0 +1,115 @@
+package database
+
+import (
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/config"
+	"github.com/ewhal/nyaa/db/postgres"
+	//"github.com/ewhal/nyaa/db/sqlite"
+	"github.com/ewhal/nyaa/model"
+	"github.com/ewhal/nyaa/util/log"
+
+	"database/sql"
+	"errors"
+)
+
+// Database obstraction layer
+type Database interface {
+
+	// Initialize internal state
+	Init() error
+
+	// return true if we need to call MigrateNext again
+	NeedsMigrate() (bool, error)
+	// migrate to next database revision
+	MigrateNext() error
+
+	// get torrents given parameters
+	GetTorrentsWhere(param *common.TorrentParam) ([]model.Torrent, error)
+
+	// insert new comment
+	InsertComment(comment *model.Comment) error
+	// new torrent report
+	InsertTorrentReport(report *model.TorrentReport) error
+
+	// check if user A follows B (by id)
+	UserFollows(a, b uint32) (bool, error)
+
+	// delete reports given params
+	DeleteTorrentReportsWhere(param *common.ReportParam) (uint32, error)
+
+	// get reports given params
+	GetTorrentReportsWhere(param *common.ReportParam) ([]model.TorrentReport, error)
+
+	// bulk record scrape events in 1 transaction
+	RecordScrapes(scrapes []common.ScrapeResult) error
+
+	// insert new user
+	InsertUser(u *model.User) error
+
+	// update existing user info
+	UpdateUser(u *model.User) error
+
+	// get users given paramteters
+	GetUsersWhere(param *common.UserParam) ([]model.User, error)
+
+	// delete many users given parameters
+	DeleteUsersWhere(param *common.UserParam) (uint32, error)
+
+	// get comments by given parameters
+	GetCommentsWhere(param *common.CommentParam) ([]model.Comment, error)
+
+	// delete comment by given parameters
+	DeleteCommentsWhere(param *common.CommentParam) (uint32, error)
+
+	// add user A following B
+	AddUserFollowing(a, b uint32) error
+
+	// delete user A following B
+	DeleteUserFollowing(a, b uint32) (bool, error)
+
+	// delete torrents by given parameters
+	DeleteTorrentsWhere(param *common.TorrentParam) (uint32, error)
+
+	// insert/update torrent
+	UpsertTorrent(t *model.Torrent) error
+
+	// DO NOT USE ME kthnx
+	Query(query string, params ...interface{}) (*sql.Rows, error)
+}
+
+var ErrInvalidDatabaseDialect = errors.New("invalid database dialect")
+var ErrSqliteSucksAss = errors.New("sqlite3 sucks ass so it's not supported yet")
+
+var Impl Database
+
+func Configure(conf *config.Config) (err error) {
+	switch conf.DBType {
+	case "postgres":
+		Impl, err = postgres.New(conf.DBParams)
+		break
+	case "sqlite3":
+		err = ErrSqliteSucksAss
+		// Impl, err = sqlite.New(conf.DBParams)
+		break
+	default:
+		err = ErrInvalidDatabaseDialect
+	}
+	if err == nil {
+		log.Infof("Init %s database", conf.DBType)
+		err = Impl.Init()
+	}
+	return
+}
+
+// Migrate migrates the database to latest revision, call after Configure
+func Migrate() (err error) {
+	next := true
+	for err == nil && next {
+		next, err = Impl.NeedsMigrate()
+		if next {
+			err = Impl.MigrateNext()
+		}
+
+	}
+	return
+}

database/postgres/comments.go

@@ -0,0 +1,21 @@
+package postgres
+
+import (
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/model"
+)
+
+func (db *Database) InsertComment(comment *model.Comment) (err error) {
+	_, err = db.getPrepared(queryInsertComment).Exec(comment.ID, comment.TorrentID, comment.Content, comment.CreatedAt)
+	return
+}
+
+func (db *Database) GetCommentsWhere(param *common.CommentParam) (comments []model.Comment, err error) {
+
+	return
+}
+
+func (db *Database) DeleteCommentsWhere(param *common.CommentParam) (deleted uint32, err error) {
+
+	return
+}

database/postgres/create_table.go

@@ -0,0 +1,91 @@
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+)
+
+// sql query
+type sqlQuery struct {
+	query  string
+	params []interface{}
+}
+
+func (q *sqlQuery) Exec(conn *sql.DB) (err error) {
+	_, err = conn.Exec(q.query, q.params...)
+	return
+}
+
+func (q *sqlQuery) QueryRow(conn *sql.DB, visitor func(*sql.Row) error) (err error) {
+	err = visitor(conn.QueryRow(q.query, q.params))
+	return
+}
+
+func (q *sqlQuery) Query(conn *sql.DB, visitor func(*sql.Rows) error) (err error) {
+	var rows *sql.Rows
+	rows, err = conn.Query(q.query, q.params...)
+	if err == sql.ErrNoRows {
+		err = nil
+	} else if err == nil {
+		err = visitor(rows)
+		rows.Close()
+	}
+	return
+}
+
+// make a createQuery that creates an index for column on table
+func createIndex(table, column string) sqlQuery {
+	return sqlQuery{
+		query: fmt.Sprintf("CREATE INDEX IF NOT EXISTS ON %s %s ", table, column),
+	}
+}
+
+// make a createQuery that creates a trigraph index on a table for multiple columns
+func createTrigraph(table string, columns ...string) sqlQuery {
+	return sqlQuery{
+		query: fmt.Sprintf("CREATE INDEX IF NOT EXISTS ON %s USING gin(%s gin_trgm_ops)", table, strings.Join(columns, ", ")),
+	}
+}
+
+// defines table creation info
+type createTable struct {
+	name       string       // Table's name
+	columns    tableColumns // Table's columns
+	preCreate  []sqlQuery   // Queries to run before table is ensrued
+	postCreate []sqlQuery   // Queries to run after table is ensured
+}
+
+func (t createTable) String() string {
+	return fmt.Sprintf("CREATE TABLE %s IF NOT EXISTS ( %s )", t.name, t.columns)
+}
+
+func (t createTable) Exec(conn *sql.DB) (err error) {
+	// pre queries
+	for idx := range t.preCreate {
+		err = t.preCreate[idx].Exec(conn)
+		if err != nil {
+			return
+		}
+	}
+	// table definition
+	_, err = conn.Exec(t.String())
+	if err != nil {
+		return
+	}
+	// post queries
+	for idx := range t.postCreate {
+		err = t.postCreate[idx].Exec(conn)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+// tableColumns is a list of columns for a table to be created
+type tableColumns []string
+
+func (def tableColumns) String() string {
+	return strings.Join(def, ", ")
+}

database/postgres/migrate.go

@@ -0,0 +1,9 @@
+package postgres
+
+func (db *Database) MigrateNext() (err error) {
+	return
+}
+
+func (db *Database) NeedsMigrate() (needs bool, err error) {
+	return
+}

database/postgres/migrate_v0.go

@@ -0,0 +1 @@
+package postgres

database/postgres/postgres.go

@@ -0,0 +1,88 @@
+package postgres
+
+import (
+	"database/sql"
+
+	"github.com/ewhal/nyaa/util/log"
+	_ "github.com/lib/pq"
+)
+
+func New(param string) (db *Database, err error) {
+	db = new(Database)
+	db.conn, err = sql.Open("postgres", param)
+	if err != nil {
+		db = nil
+	}
+	return
+}
+
+type Database struct {
+	conn     *sql.DB
+	prepared map[string]*sql.Stmt
+}
+
+func (db *Database) getPrepared(name string) *sql.Stmt {
+	return db.prepared[name]
+}
+
+func (db *Database) Query(q string, param ...interface{}) (*sql.Rows, error) {
+	return db.conn.Query(q, param)
+}
+
+func (db *Database) Init() (err error) {
+
+	// ensure tables
+	for idx := range tables {
+		log.Debugf("ensure table %s", tables[idx].name)
+		err = tables[idx].Exec(db.conn)
+		if err != nil {
+			log.Errorf("Failed to ensure table %s: %s", tables[idx].name, err.Error())
+			return
+		}
+	}
+
+	// generate prepared statements
+	for k := range statements {
+		var stmt *sql.Stmt
+		stmt, err = db.conn.Prepare(statements[k])
+		if err != nil {
+			log.Errorf("failed to build prepared statement %s: %s", k, err.Error())
+			return
+		}
+		db.prepared[k] = stmt
+	}
+	return
+}
+
+// execute prepared statement with arguments, visit result and autoclose rows after done visiting
+func (db *Database) queryWithPrepared(name string, visit func(*sql.Rows) error, params ...interface{}) (err error) {
+	var rows *sql.Rows
+	rows, err = db.getPrepared(name).Query(params)
+	if err == sql.ErrNoRows {
+		err = nil
+	} else if err == nil {
+		err = visit(rows)
+		rows.Close()
+	}
+	return
+}
+
+// execute prepared statement with arguments, visit single row
+func (db *Database) queryRowWithPrepared(name string, visit func(*sql.Row) error, params ...interface{}) (err error) {
+	err = visit(db.getPrepared(name).QueryRow(params))
+	return
+}
+
+// execute a query by name and return how many rows were affected
+func (db *Database) execQuery(name string, p ...interface{}) (affected uint32, err error) {
+	var result sql.Result
+	result, err = db.getPrepared(name).Exec(p)
+	if err == nil {
+		var d int64
+		d, err = result.RowsAffected()
+		if err == nil {
+			affected = uint32(d)
+		}
+	}
+	return
+}

database/postgres/report.go

@@ -0,0 +1,64 @@
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/model"
+)
+
+func (db *Database) InsertTorrentReport(report *model.TorrentReport) (err error) {
+	_, err = db.getPrepared(queryInsertTorrentReport).Exec(report.Type, report.TorrentID, report.UserID, report.CreatedAt)
+	return
+}
+
+func reportParamToQuery(param *common.ReportParam) (q sqlQuery) {
+	q.query += fmt.Sprintf("SELECT %s FROM %s WHERE created_at IS NOT NULL ", torrentReportSelectColumnsFull, tableTorrentReports)
+
+	counter := 1
+
+	if !param.AllTime {
+		q.query += fmt.Sprintf("AND created_at < $%d AND created_at > $%d", counter, counter+1)
+		q.params = append(q.params, param.Before, param.After)
+		counter += 2
+	}
+
+	if param.Limit > 0 {
+		q.query += fmt.Sprintf("LIMIT $%d ", counter)
+		q.params = append(q.params, param.Limit)
+		counter++
+	}
+	if param.Offset > 0 {
+		q.query += fmt.Sprintf("OFFSET $%d ", counter)
+		q.params = append(q.params, param.Offset)
+		counter++
+	}
+	return
+}
+
+func (db *Database) GetTorrentReportsWhere(param *common.ReportParam) (reports []model.TorrentReport, err error) {
+	q := reportParamToQuery(param)
+	err = q.Query(db.conn, func(rows *sql.Rows) error {
+		for rows.Next() {
+			var r model.TorrentReport
+			scanTorrentReportColumnsFull(rows, &r)
+			reports = append(reports, r)
+		}
+		return nil
+	})
+	return
+}
+
+func (db *Database) DeleteTorrentReportByID(id uint32) (err error) {
+	_, err = db.getPrepared(queryDeleteTorrentReportByID).Exec(id)
+	if err == sql.ErrNoRows {
+		err = nil
+	}
+	return
+}
+
+func (db *Database) DeleteTorrentReportsWhere(param *common.ReportParam) (deleted uint32, err error) {
+
+	return
+}

database/postgres/scrape.go

@@ -0,0 +1,28 @@
+package postgres
+
+import (
+	"database/sql"
+	"github.com/ewhal/nyaa/common"
+)
+
+func (db *Database) RecordScrapes(scrape []common.ScrapeResult) (err error) {
+	if len(scrape) > 0 {
+		var tx *sql.Tx
+		tx, err = db.conn.Begin()
+		if err == nil {
+			st := tx.Stmt(db.getPrepared(queryInsertScrape))
+			for idx := range scrape {
+				_, err = st.Exec(scrape[idx].Seeders, scrape[idx].Leechers, scrape[idx].Completed, scrape[idx].Date, scrape[idx].TorrentID)
+				if err != nil {
+					break
+				}
+			}
+		}
+		if err == nil {
+			err = tx.Commit()
+		} else {
+			tx.Rollback()
+		}
+	}
+	return
+}

database/postgres/statements.go

@@ -0,0 +1,73 @@
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/ewhal/nyaa/model"
+)
+
+const queryGetAllTorrents = "GetAllTorrents"
+const queryGetTorrentByID = "GetTorrentByID"
+const queryInsertComment = "InsertComment"
+const queryInsertUser = "InsertUser"
+const queryInsertTorrentReport = "InsertTorrentReport"
+const queryUserFollows = "UserFollows"
+const queryDeleteTorrentReportByID = "DeleteTorrentReportByID"
+const queryInsertScrape = "InsertScrape"
+const queryGetUserByApiToken = "GetUserByApiToken"
+const queryGetUserByEmail = "GetUserByEmail"
+const queryGetUserByName = "GetUserByName"
+const queryGetUserByID = "GetUserByID"
+const queryUpdateUser = "UpdateUser"
+const queryDeleteUserByID = "DeleteUserByID"
+const queryDeleteUserByEmail = "DeleteUserByEmail"
+const queryDeleteUserByToken = "DeleteUserByToken"
+const queryUserFollowsUpsert = "UserFollowsUpsert"
+const queryDeleteUserFollowing = "DeleteUserFollowing"
+
+const torrentSelectColumnsFull = `torrent_id, torrent_name, torrent_hash, category, sub_category, status, date, uploader, downloads, stardom, description, website_link, deleted_at, seeders, leechers, completed, last_scrape`
+
+func scanTorrentColumnsFull(rows *sql.Rows, t *model.Torrent) {
+	rows.Scan(&t.ID, &t.Name, &t.Hash, &t.Category, &t.SubCategory, &t.Status, &t.Date, &t.UploaderID, &t.Downloads, &t.Stardom, &t.Description, &t.WebsiteLink, &t.DeletedAt, &t.Seeders, &t.Leechers, &t.Completed, &t.LastScrape)
+}
+
+const commentSelectColumnsFull = `comment_id, torrent_id, user_id, content, created_at, updated_at, deleted_at`
+
+func scanCommentColumnsFull(rows *sql.Rows, c *model.Comment) {
+
+}
+
+const torrentReportSelectColumnsFull = `torrent_report_id, type, torrent_id, user_id, created_at`
+
+func scanTorrentReportColumnsFull(rows *sql.Rows, r *model.TorrentReport) {
+	rows.Scan(&r.ID, &r.Type, &r.TorrentID, &r.UserID, &r.CreatedAt)
+}
+
+const userSelectColumnsFull = `user_id, username, password, email, status, created_at, updated_at, last_login_at, last_login_ip, api_token, api_token_expires, language, md5`
+
+func scanUserColumnsFull(rows *sql.Rows, u *model.User) {
+	rows.Scan(&u.ID, &u.Username, &u.Password, &u.Email, &u.Status, &u.CreatedAt, &u.UpdatedAt, &u.LastLoginAt, &u.LastLoginIP, &u.Token, &u.TokenExpiration, &u.Language, &u.MD5)
+
+}
+
+var statements = map[string]string{
+	queryGetTorrentByID:          fmt.Sprintf("SELECT %s FROM %s WHERE torrent_id = $1 LIMIT 1", torrentSelectColumnsFull, tableTorrents),
+	queryGetAllTorrents:          fmt.Sprintf("SELECT %s FROM %s LIMIT $2 OFFSET $1", torrentSelectColumnsFull, tableTorrents),
+	queryInsertComment:           fmt.Sprintf("INSERT INTO %s (comment_id, torrent_id, content, created_at) VALUES ($1, $2, $3, $4)", tableComments),
+	queryInsertTorrentReport:     fmt.Sprintf("INSERT INTO %s (type, torrent_id, user_id, created_at) VALUES ($1, $2, $3, $4)", tableTorrentReports),
+	queryUserFollows:             fmt.Sprintf("SELECT user_id, following FROM %s WHERE user_id = $1 AND following = $1 LIMIT 1", tableUserFollows),
+	queryDeleteTorrentReportByID: fmt.Sprintf("DELETE FROM %s WHERE torrent_report_id = $1", tableTorrentReports),
+	queryInsertScrape:            fmt.Sprintf("UPDATE %s SET (seeders = $1, leechers = $2, completed = $3, last_scrape = $4 ) WHERE torrent_id = $5", tableTorrents),
+	queryGetUserByApiToken:       fmt.Sprintf("SELECT %s FROM %s WHERE api_token = $1", userSelectColumnsFull, tableUsers),
+	queryGetUserByEmail:          fmt.Sprintf("SELECT %s FROM %s WHERE email = $1", userSelectColumnsFull, tableUsers),
+	queryGetUserByName:           fmt.Sprintf("SELECT %s FROM %s WHERE username = $1", userSelectColumnsFull, tableUsers),
+	queryGetUserByID:             fmt.Sprintf("SELECT %s FROM %s WHERE user_id = $1", userSelectColumnsFull, tableUsers),
+	queryInsertUser:              fmt.Sprintf("INSERT INTO %s (username, password, email, status, created_at, updated_at, last_login_at, last_login_ip, api_token, api_token_expires, language, md5 ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", tableUsers),
+	queryUpdateUser:              fmt.Sprintf("UPDATE %s SET (username = $2, password = $3 , email = $4, status = $5, updated_at = $6, last_login_at = $7 , last_login_ip = $8 , api_token = $9 , api_token_expiry = $10 , language = $11 , md5 = $12 ) WHERE user_id = $1", tableUsers),
+	queryDeleteUserByID:          fmt.Sprintf("DELETE FROM %s WHERE user_id = $1", tableUsers),
+	queryDeleteUserByEmail:       fmt.Sprintf("DELETE FROM %s WHERE email = $1", tableUsers),
+	queryDeleteUserByToken:       fmt.Sprintf("DELETE FROM %s WHERE api_token = $1", tableUsers),
+	queryUserFollowsUpsert:       fmt.Sprintf("INSERT INTO %s VALUES(user_id, following) ($1, $2) ON CONFLICT DO UPDATE", tableUserFollows),
+	queryDeleteUserFollowing:     fmt.Sprintf("DELETE FROM %s WHERE user_id = $1 AND following = $2", tableUserFollows),
+}

database/postgres/tables.go

@@ -0,0 +1,141 @@
+package postgres
+
+import "fmt"
+
+// table name for torrents
+const tableTorrents = "torrents"
+
+// table name for users
+const tableUsers = "users"
+
+// table for new comments
+const tableComments = "comments"
+
+// table for old comments
+const tableOldComments = "comments_old"
+
+// table for torrent reports
+const tableTorrentReports = "torrent_reports"
+
+// table for user follows
+const tableUserFollows = "user_follows"
+
+// table for old user uploads
+const tableOldUserUploads = "user_uploads_old"
+
+// all tables that we have in current database schema in the order they are created
+var tables = []createTable{
+	// users table
+	createTable{
+		name: tableUsers,
+		columns: tableColumns{
+			"user_id SERIAL PRIMARY KEY",
+			"username TEXT NOT NULL",
+			"password TEXT NOT NULL",
+			"email TEXT",
+			"status INTEGER NOT NULL",
+			"created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL",
+			"updated_at TIMESTAMP WITHOUT TIME ZONE",
+			"last_login_at TIMESTAMP WITHOUT TIME ZONE",
+			"last_login_ip TEXT",
+			"api_token TEXT",
+			"api_token_expiry TIMESTAMP WITHOUT TIME ZONE NOT NULL",
+			"language TEXT",
+			"MD5 TEXT",
+		},
+		postCreate: []sqlQuery{
+			createIndex(tableUsers, "username"),
+		},
+	},
+	// torrents table
+	createTable{
+		name: tableTorrents,
+		columns: tableColumns{
+			"torrent_id SERIAL PRIMARY KEY",
+			"torrent_name TEXT",
+			"torrent_hash TEXT NOT NULL",
+			"category INTEGER NOT NULL",
+			"sub_category INTEGER NOT NULL",
+			"status INTEGER NOT NULL",
+			"date TIMESTAMP WITHOUT TIME ZONE",
+			fmt.Sprintf("uploader INTEGER NOT NULL REFERENCES %s (user_id)", tableUsers),
+			"downloads INTEGER",
+			"stardom INTEGER NOT NULL",
+			"filesize BIGINT",
+			"description TEXT NOT NULL",
+			"website_link TEXT",
+			"deleted_at TIMESTAMP WITHOUT TIME ZONE",
+			"seeders INTEGER",
+			"leechers INTEGER",
+			"completed INTEGER",
+			"last_scrape TIMESTAMP WITHOUT TIME ZONE",
+		},
+		postCreate: []sqlQuery{
+			createIndex(tableTorrents, "torrent_id"),
+			createIndex(tableTorrents, "deleted_at"),
+			createIndex(tableTorrents, "uploader"),
+			createTrigraph(tableTorrents, "category", "torrent_name"),
+			createTrigraph(tableTorrents, "sub_category", "torrent_name"),
+			createTrigraph(tableTorrents, "status", "torrent_name"),
+			createTrigraph(tableTorrents, "torrent_name"),
+		},
+	},
+	// new comments table
+	createTable{
+		name: tableComments,
+		columns: tableColumns{
+			"comment_id SERIAL PRIMARY KEY",
+			fmt.Sprintf("torrent_id INTEGER REFERENCES %s (torrent_id)", tableTorrents),
+			fmt.Sprintf("user_id INTEGER REFERENCES %s (user_id)", tableUsers),
+			"content TEXT NOT NULL",
+			"created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL",
+			"updated_at TIMESTAMP WITHOUT TIME ZONE",
+			"deleted_at TIMESTAMP WITH TIME ZONE",
+		},
+		postCreate: []sqlQuery{
+			createIndex(tableComments, "torrent_id"),
+		},
+	},
+	// old comments table
+	createTable{
+		name: tableOldComments,
+		columns: tableColumns{
+			fmt.Sprintf("torrent_id INTEGER NOT NULL REFERENCES %s (torrent_id)", tableTorrents),
+			"username TEXT NOT NULL",
+			"content TEXT NOT NULL",
+			"date TIMESTAMP WITHOUT TIME ZONE NOT NULL",
+		},
+	},
+	// torrent reports table
+	createTable{
+		name: tableTorrentReports,
+		columns: tableColumns{
+			"torrent_report_id SERIAL PRIMARY KEY",
+			"type TEXT",
+			fmt.Sprintf("torrent_id INTEGER REFERENCES %s (torrent_id)", tableTorrents),
+			"user_id INTEGER",
+			"created_at TIMESTAMP WITH TIME ZONE",
+		},
+		postCreate: []sqlQuery{
+			createIndex(tableTorrentReports, "torrent_report_id"),
+		},
+	},
+	// user follows table
+	createTable{
+		name: tableUserFollows,
+		columns: tableColumns{
+			"user_id INTEGER NOT NULL",
+			"following INTEGER NOT NULL",
+			"PRIMARY KEY(user_id, following)",
+		},
+	},
+	// old uploads table
+	createTable{
+		name: tableOldUserUploads,
+		columns: tableColumns{
+			"username TEXT IS NOT NULL",
+			fmt.Sprintf("torrent_id INTEGER IS NOT NULL REFERENCES %s (torrent_id)", tableTorrents),
+			"PRIMARY KEY (torrent_id)",
+		},
+	},
+}

database/postgres/torrent.go

@@ -0,0 +1,33 @@
+package postgres
+
+import (
+	"github.com/ewhal/nyaa/model"
+
+	"database/sql"
+)
+
+func (db *Database) GetAllTorrents(offset, limit uint32) (torrents []model.Torrent, err error) {
+	err = db.queryWithPrepared(queryGetAllTorrents, func(rows *sql.Rows) error {
+		torrents = make([]model.Torrent, 0, limit)
+		var idx uint64
+		for rows.Next() {
+			rows.Scan(torrents[idx])
+		}
+		return nil
+	}, offset, limit)
+	return
+}
+
+func (db *Database) GetTorrentByID(id uint32) (torrent model.Torrent, has bool, err error) {
+	err = db.queryWithPrepared(queryGetTorrentByID, func(rows *sql.Rows) error {
+		rows.Next()
+		scanTorrentColumnsFull(rows, &torrent)
+		has = true
+		return nil
+	}, id)
+	return
+}
+
+func (db *Database) UpsertTorrent(t *model.Torrent) (err error) {
+	return
+}

database/postgres/user.go

@@ -0,0 +1,153 @@
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/model"
+)
+
+func userParamToSelectQuery(p *common.UserParam) (q sqlQuery) {
+	q.query = fmt.Sprintf("SELECT %s FROM %s ", userSelectColumnsFull, tableUsers)
+	counter := 1
+	if p.Max > 0 {
+		q.query += fmt.Sprintf("LIMIT $%d ", counter)
+		q.params = append(q.params, p.Max)
+		counter++
+	}
+	if p.Offset > 0 {
+		q.query += fmt.Sprintf("OFFSET $%d ", counter)
+		q.params = append(q.params, p.Offset)
+		counter++
+	}
+	return
+}
+
+func (db *Database) UserFollows(a, b uint32) (follows bool, err error) {
+	err = db.queryWithPrepared(queryUserFollows, func(rows *sql.Rows) error {
+		follows = true
+		return nil
+	}, a, b)
+	return
+}
+
+func (db *Database) AddUserFollowing(a, b uint32) (err error) {
+	_, err = db.getPrepared(queryUserFollowsUpsert).Exec(a, b)
+	return
+}
+
+func (db *Database) DeleteUserFollowing(a, b uint32) (deleted bool, err error) {
+	var affected uint32
+	affected, err = db.execQuery(queryDeleteUserFollowing, a, b)
+	deleted = affected > 0
+	return
+}
+
+func (db *Database) getUserByQuery(name string, p interface{}) (user model.User, has bool, err error) {
+	err = db.queryWithPrepared(name, func(rows *sql.Rows) error {
+		rows.Next()
+		scanUserColumnsFull(rows, &user)
+		has = true
+		return nil
+	}, p)
+	return
+}
+
+func (db *Database) GetUserByAPIToken(token string) (user model.User, has bool, err error) {
+	user, has, err = db.getUserByQuery(queryGetUserByApiToken, token)
+	return
+}
+
+func (db *Database) GetUsersByEmail(email string) (users []model.User, err error) {
+	err = db.queryWithPrepared(queryGetUserByEmail, func(rows *sql.Rows) error {
+		for rows.Next() {
+			var user model.User
+			scanUserColumnsFull(rows, &user)
+			users = append(users, user)
+		}
+		return nil
+	}, email)
+	return
+}
+
+func (db *Database) GetUserByName(name string) (user model.User, has bool, err error) {
+	user, has, err = db.getUserByQuery(queryGetUserByName, name)
+	return
+}
+
+func (db *Database) GetUserByID(id uint32) (user model.User, has bool, err error) {
+	user, has, err = db.getUserByQuery(queryGetUserByID, id)
+	return
+}
+
+func (db *Database) InsertUser(u *model.User) (err error) {
+	_, err = db.getPrepared(queryInsertUser).Exec(u.Username, u.Password, u.Email, u.Status, u.CreatedAt, u.UpdatedAt, u.LastLoginAt, u.LastLoginIP, u.Token, u.TokenExpiration, u.Language, u.MD5)
+	return
+}
+
+func (db *Database) UpdateUser(u *model.User) (err error) {
+	_, err = db.getPrepared(queryUpdateUser).Exec(u.ID, u.Username, u.Password, u.Email, u.Status, u.UpdatedAt, u.LastLoginAt, u.LastLoginIP, u.Token, u.TokenExpiration, u.Language, u.MD5)
+	return
+}
+
+func (db *Database) GetUsersWhere(param *common.UserParam) (users []model.User, err error) {
+	var user model.User
+	var has bool
+	if len(param.Email) > 0 {
+		users, err = db.GetUsersByEmail(param.Email)
+	} else if len(param.Name) > 0 {
+		user, has, err = db.GetUserByName(param.Name)
+		if has {
+			users = append(users, user)
+		}
+	} else if len(param.ApiToken) > 0 {
+		user, has, err = db.GetUserByAPIToken(param.ApiToken)
+		if has {
+			users = append(users, user)
+		}
+	} else if param.ID > 0 {
+		user, has, err = db.GetUserByID(param.ID)
+		if has {
+			users = append(users, user)
+		}
+	} else {
+		q := userParamToSelectQuery(param)
+		if param.Max > 0 {
+			users = make([]model.User, 0, param.Max)
+		} else {
+			users = make([]model.User, 0, 64)
+		}
+		err = q.Query(db.conn, func(rows *sql.Rows) error {
+
+			for rows.Next() {
+				var user model.User
+				scanUserColumnsFull(rows, &user)
+				users = append(users, user)
+			}
+			return nil
+		})
+	}
+	return
+}
+
+func (db *Database) DeleteUsersWhere(param *common.UserParam) (deleted uint32, err error) {
+
+	var queryName string
+	var p interface{}
+	if param.ID > 0 {
+		queryName = queryDeleteUserByID
+		p = param.ID
+	} else if len(param.Email) > 0 {
+		queryName = queryDeleteUserByEmail
+		p = param.Email
+	} else if len(param.ApiToken) > 0 {
+		queryName = queryDeleteUserByToken
+		p = param.ApiToken
+	} else {
+		// delete nothing
+		return
+	}
+	deleted, err = db.execQuery(queryName, p)
+	return
+}

database/postgres/where.go

@@ -0,0 +1,169 @@
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"github.com/ewhal/nyaa/common"
+	"github.com/ewhal/nyaa/model"
+)
+
+// build sql query from SearchParam for torrent search
+func searchParamToTorrentQuery(param *common.TorrentParam) (q sqlQuery) {
+	counter := 1
+	q.query = fmt.Sprintf("SELECT %s FROM %s ", torrentSelectColumnsFull, tableTorrents)
+	if param.Category.IsSet() {
+		q.query += fmt.Sprintf("WHERE category = $%d AND sub_category = $%d ", counter, counter+1)
+		q.params = append(q.params, param.Category.Main, param.Category.Sub)
+		counter += 2
+	}
+	if counter > 1 {
+		q.query += "AND "
+	} else {
+		q.query += "WHERE "
+	}
+
+	q.query += fmt.Sprintf("status >= $%d ", counter)
+	q.params = append(q.params, param.Status)
+	counter++
+	if param.UserID > 0 {
+		q.query += fmt.Sprintf("AND uploader = $%d ", counter)
+		q.params = append(q.params, param.UserID)
+		counter++
+	}
+
+	notnulls := strings.Split(param.NotNull, ",")
+	for idx := range notnulls {
+		k := strings.ToLower(strings.TrimSpace(notnulls[idx]))
+		switch k {
+		case "date":
+		case "downloads":
+		case "filesize":
+		case "website_link":
+		case "deleted_at":
+		case "seeders":
+		case "leechers":
+		case "completed":
+		case "last_scrape":
+			q.query += fmt.Sprintf("AND %s IS NOT NULL ", k)
+			break
+		default:
+			break
+		}
+	}
+
+	nulls := strings.Split(param.Null, ",")
+	for idx := range nulls {
+		k := strings.ToLower(strings.TrimSpace(nulls[idx]))
+		switch k {
+		case "date":
+		case "downloads":
+		case "filesize":
+		case "website_link":
+		case "deleted_at":
+		case "seeders":
+		case "leechers":
+		case "completed":
+		case "last_scrape":
+			q.query += fmt.Sprintf("AND %s IS NULL ", k)
+			break
+		default:
+			break
+		}
+	}
+
+	nameLikes := strings.Split(param.NameLike, ",")
+	for idx := range nameLikes {
+		q.query += fmt.Sprintf("AND torrent_name ILIKE $%d", counter)
+		q.params = append(q.params, strings.TrimSpace(nameLikes[idx]))
+		counter++
+	}
+
+	var sort string
+	switch param.Sort {
+	case common.Name:
+		sort = "torrent_name"
+		break
+	case common.Date:
+		sort = "date"
+		break
+	case common.Downloads:
+		sort = "downloads"
+		break
+	case common.Size:
+		sort = "filesize"
+		break
+	case common.Seeders:
+		sort = "seeders"
+		break
+	case common.Leechers:
+		sort = "leechers"
+		break
+	case common.Completed:
+		sort = "completed"
+		break
+	case common.ID:
+	default:
+		sort = "torrent_id"
+	}
+
+	q.query += fmt.Sprintf("ORDER BY %s ", sort)
+	if param.Order {
+		q.query += "ASC "
+	} else {
+		q.query += "DESC "
+	}
+
+	if param.Max > 0 {
+		q.query += fmt.Sprintf("LIMIT $%d ", counter)
+		q.params = append(q.params, param.Max)
+		counter++
+	}
+	if param.Offset > 0 {
+		q.query += fmt.Sprintf("OFFSET $%d ", counter)
+		q.params = append(q.params, param.Offset)
+		counter++
+	}
+	return
+}
+
+func (db *Database) GetTorrentsWhere(param *common.TorrentParam) (torrents []model.Torrent, err error) {
+	if param.TorrentID > 0 {
+		var torrent model.Torrent
+		var has bool
+		torrent, has, err = db.GetTorrentByID(param.TorrentID)
+		if has {
+			torrents = append(torrents, torrent)
+		}
+		return
+	}
+
+	if param.All {
+		torrents, err = db.GetAllTorrents(param.Offset, param.Max)
+		return
+	}
+
+	q := searchParamToTorrentQuery(param)
+	err = q.Query(db.conn, func(rows *sql.Rows) error {
+		if param.Max == 0 {
+			torrents = make([]model.Torrent, 0, 128)
+		} else {
+			torrents = make([]model.Torrent, 0, param.Max)
+		}
+
+		for rows.Next() {
+			var t model.Torrent
+			scanTorrentColumnsFull(rows, &t)
+			torrents = append(torrents, t)
+			// grow as needed
+			if len(torrents) >= cap(torrents) {
+				newtorrents := make([]model.Torrent, cap(torrents), cap(torrents)*3/2) // XXX: adjust as needed
+				copy(newtorrents, torrents)
+				torrents = newtorrents
+			}
+		}
+		return nil
+	})
+	return
+}

database/sqlite/sqlite.go

@@ -0,0 +1,29 @@
+package sqlite
+
+import (
+	"database/sql"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+// queryEvent is a queued event to be executed in a pipeline to ensure that sqlite access is done from 1 goroutine
+type queryEvent struct {
+	query        string
+	param        []interface{}
+	handleResult func(*sql.Rows, error)
+}
+
+func New(param string) (db *Database, err error) {
+	db = new(Database)
+	db.conn, err = sql.Open("sqlite3", param)
+	if err == nil {
+		db.query = make(chan *queryEvent, 128)
+	} else {
+		db = nil
+	}
+	return
+}
+
+type Database struct {
+	conn  *sql.DB
+	query chan *queryEvent
+}