* [WIP] Torrent Generation on not found error As asked in #1517, it allows on-the-fly torrent generation. Since it uses magnet links, it needs some time to connect to peers. So it can't be instant generation, we need the user to wait and try after a minute at least. * Replace Fatal by simple error * attempt at fixing travis * del * Add Anacrolyx dependency * Add back difflib * Remove .torrent suffix in the url example * Add some explanations when file missing page shown * Ignore downloads directory * Either use cache (third-party site) or own download directory * Wrong import * If there is an error then it means we aren't generating a torrent file May it be "torrent not found" or "We do not store torrent files" which are the two only existing errors for this page * hash is never empty * TorrentLink may be empty at times So we add a /download/:hash link if it is * Update README.md * Made a mistake here, need to check if false * Update en-us.all.json * Update CHANGELOG.md * Torrent file generation can be triggered by click on button if JS enabled * Update download.go * Update download.go * Use c.JSON instead of text/template * Return to default behavior if we don't generate the file * Don't do the query if returned to default behavior * Add "Could not generate torrent file" error * Fix JS condition & lower delay until button updates * Start download automatically once torrent file is generated * Fix torrentFileExists() constantly returning false if external torrent download URL * torrent-view-data is two tables instead of one This allows the removal of useless things without any problem (e.g Website link), but also a better responsibe design since the previous one separated stats after a certain res looking very wonky * CSS changes to go along * Remove useless <b></b> * Update main.css * In torrentFileExists, check if filestorage path exists instead of looking at the domain in torrent link When checking if the file is stored on another server i used to simply check if the domain name was inside the torrent link, but we can straight up check for filestorage length * Fix JS of on-demand stat fetching * ScrapeAge variable accessible through view.jet.html Contains last scraped time in hours, is at -1 is torrent has never been scraped Stats will get updated if it's either at -1 or above 1460 (2 months old) * Refresh stats if older than two months OR unknown and older than 24h Show last scraped date even if stats are unknown * Add StatsObsolete variable to torrent Indicating if: - They can be shown - They need to be updated * Update scraped data even if Unknown, prevent users from trying to fetch stats every seconds * Torrent file stored locally by default * no need to do all of that if no filestorage * fix filestorage path * Fix torrent download button stuck on "Generating torrent file" at rare times * fix some css rules that didn't work on IE * Fix panic error Seems like this error is a known bug from anacrolyx torrent https://github.com/anacrolix/torrent/issues/83 To prevent it, I'm creating a single client and modifying the socket.go to make it not raise a panic but a simple error log.
375 lignes
10 Kio
Go
375 lignes
10 Kio
Go
package unsnap
|
|
|
|
// copyright (c) 2014, Jason E. Aten
|
|
// license: MIT
|
|
|
|
// Some text from the Golang standard library doc is adapted and
|
|
// reproduced in fragments below to document the expected behaviors
|
|
// of the interface functions Read()/Write()/ReadFrom()/WriteTo() that
|
|
// are implemented here. Those descriptions (see
|
|
// http://golang.org/pkg/io/#Reader for example) are
|
|
// copyright 2010 The Go Authors.
|
|
|
|
import "io"
|
|
|
|
// FixedSizeRingBuf:
|
|
//
|
|
// a fixed-size circular ring buffer. Yes, just what is says.
|
|
//
|
|
// We keep a pair of ping/pong buffers so that we can linearize
|
|
// the circular buffer into a contiguous slice if need be.
|
|
//
|
|
// For efficiency, a FixedSizeRingBuf may be vastly preferred to
|
|
// a bytes.Buffer. The ReadWithoutAdvance(), Advance(), and Adopt()
|
|
// methods are all non-standard methods written for speed.
|
|
//
|
|
// For an I/O heavy application, I have replaced bytes.Buffer with
|
|
// FixedSizeRingBuf and seen memory consumption go from 8GB to 25MB.
|
|
// Yes, that is a 300x reduction in memory footprint. Everything ran
|
|
// faster too.
|
|
//
|
|
// Note that Bytes(), while inescapable at times, is expensive: avoid
|
|
// it if possible. Instead it is better to use the FixedSizeRingBuf.Readable
|
|
// member to get the number of bytes available. Bytes() is expensive because
|
|
// it may copy the back and then the front of a wrapped buffer A[Use]
|
|
// into A[1-Use] in order to get a contiguous slice. If possible use ContigLen()
|
|
// first to get the size that can be read without copying, Read() that
|
|
// amount, and then Read() a second time -- to avoid the copy.
|
|
|
|
type FixedSizeRingBuf struct {
|
|
A [2][]byte // a pair of ping/pong buffers. Only one is active.
|
|
Use int // which A buffer is in active use, 0 or 1
|
|
N int // MaxViewInBytes, the size of A[0] and A[1] in bytes.
|
|
Beg int // start of data in A[Use]
|
|
Readable int // number of bytes available to read in A[Use]
|
|
|
|
OneMade bool // lazily instantiate the [1] buffer. If we never call Bytes(),
|
|
// we may never need it. If OneMade is false, the Use must be = 0.
|
|
}
|
|
|
|
func (b *FixedSizeRingBuf) Make2ndBuffer() {
|
|
if b.OneMade {
|
|
return
|
|
}
|
|
b.A[1] = make([]byte, b.N, b.N)
|
|
b.OneMade = true
|
|
}
|
|
|
|
// get the length of the largest read that we can provide to a contiguous slice
|
|
// without an extra linearizing copy of all bytes internally.
|
|
func (b *FixedSizeRingBuf) ContigLen() int {
|
|
extent := b.Beg + b.Readable
|
|
firstContigLen := intMin(extent, b.N) - b.Beg
|
|
return firstContigLen
|
|
}
|
|
|
|
func NewFixedSizeRingBuf(maxViewInBytes int) *FixedSizeRingBuf {
|
|
n := maxViewInBytes
|
|
r := &FixedSizeRingBuf{
|
|
Use: 0, // 0 or 1, whichever is actually in use at the moment.
|
|
// If we are asked for Bytes() and we wrap, linearize into the other.
|
|
|
|
N: n,
|
|
Beg: 0,
|
|
Readable: 0,
|
|
OneMade: false,
|
|
}
|
|
r.A[0] = make([]byte, n, n)
|
|
|
|
// r.A[1] initialized lazily now.
|
|
|
|
return r
|
|
}
|
|
|
|
// from the standard library description of Bytes():
|
|
// Bytes() returns a slice of the contents of the unread portion of the buffer.
|
|
// If the caller changes the contents of the
|
|
// returned slice, the contents of the buffer will change provided there
|
|
// are no intervening method calls on the Buffer.
|
|
//
|
|
func (b *FixedSizeRingBuf) Bytes() []byte {
|
|
|
|
extent := b.Beg + b.Readable
|
|
if extent <= b.N {
|
|
// we fit contiguously in this buffer without wrapping to the other
|
|
return b.A[b.Use][b.Beg:(b.Beg + b.Readable)]
|
|
}
|
|
|
|
// wrap into the other buffer
|
|
b.Make2ndBuffer()
|
|
|
|
src := b.Use
|
|
dest := 1 - b.Use
|
|
|
|
n := copy(b.A[dest], b.A[src][b.Beg:])
|
|
n += copy(b.A[dest][n:], b.A[src][0:(extent%b.N)])
|
|
|
|
b.Use = dest
|
|
b.Beg = 0
|
|
|
|
return b.A[b.Use][:n]
|
|
}
|
|
|
|
// Read():
|
|
//
|
|
// from bytes.Buffer.Read(): Read reads the next len(p) bytes
|
|
// from the buffer or until the buffer is drained. The return
|
|
// value n is the number of bytes read. If the buffer has no data
|
|
// to return, err is io.EOF (unless len(p) is zero); otherwise it is nil.
|
|
//
|
|
// from the description of the Reader interface,
|
|
// http://golang.org/pkg/io/#Reader
|
|
//
|
|
/*
|
|
Reader is the interface that wraps the basic Read method.
|
|
|
|
Read reads up to len(p) bytes into p. It returns the number
|
|
of bytes read (0 <= n <= len(p)) and any error encountered.
|
|
Even if Read returns n < len(p), it may use all of p as scratch
|
|
space during the call. If some data is available but not
|
|
len(p) bytes, Read conventionally returns what is available
|
|
instead of waiting for more.
|
|
|
|
When Read encounters an error or end-of-file condition after
|
|
successfully reading n > 0 bytes, it returns the number of bytes
|
|
read. It may return the (non-nil) error from the same call or
|
|
return the error (and n == 0) from a subsequent call. An instance
|
|
of this general case is that a Reader returning a non-zero number
|
|
of bytes at the end of the input stream may return
|
|
either err == EOF or err == nil. The next Read should
|
|
return 0, EOF regardless.
|
|
|
|
Callers should always process the n > 0 bytes returned before
|
|
considering the error err. Doing so correctly handles I/O errors
|
|
that happen after reading some bytes and also both of the
|
|
allowed EOF behaviors.
|
|
|
|
Implementations of Read are discouraged from returning a zero
|
|
byte count with a nil error, and callers should treat that
|
|
situation as a no-op.
|
|
*/
|
|
//
|
|
|
|
func (b *FixedSizeRingBuf) Read(p []byte) (n int, err error) {
|
|
return b.ReadAndMaybeAdvance(p, true)
|
|
}
|
|
|
|
// if you want to Read the data and leave it in the buffer, so as
|
|
// to peek ahead for example.
|
|
func (b *FixedSizeRingBuf) ReadWithoutAdvance(p []byte) (n int, err error) {
|
|
return b.ReadAndMaybeAdvance(p, false)
|
|
}
|
|
|
|
func (b *FixedSizeRingBuf) ReadAndMaybeAdvance(p []byte, doAdvance bool) (n int, err error) {
|
|
if len(p) == 0 {
|
|
return 0, nil
|
|
}
|
|
if b.Readable == 0 {
|
|
return 0, io.EOF
|
|
}
|
|
extent := b.Beg + b.Readable
|
|
if extent <= b.N {
|
|
n += copy(p, b.A[b.Use][b.Beg:extent])
|
|
} else {
|
|
n += copy(p, b.A[b.Use][b.Beg:b.N])
|
|
if n < len(p) {
|
|
n += copy(p[n:], b.A[b.Use][0:(extent%b.N)])
|
|
}
|
|
}
|
|
if doAdvance {
|
|
b.Advance(n)
|
|
}
|
|
return
|
|
}
|
|
|
|
//
|
|
// Write writes len(p) bytes from p to the underlying data stream.
|
|
// It returns the number of bytes written from p (0 <= n <= len(p))
|
|
// and any error encountered that caused the write to stop early.
|
|
// Write must return a non-nil error if it returns n < len(p).
|
|
//
|
|
func (b *FixedSizeRingBuf) Write(p []byte) (n int, err error) {
|
|
for {
|
|
if len(p) == 0 {
|
|
// nothing (left) to copy in; notice we shorten our
|
|
// local copy p (below) as we read from it.
|
|
return
|
|
}
|
|
|
|
writeCapacity := b.N - b.Readable
|
|
if writeCapacity <= 0 {
|
|
// we are all full up already.
|
|
return n, io.ErrShortWrite
|
|
}
|
|
if len(p) > writeCapacity {
|
|
err = io.ErrShortWrite
|
|
// leave err set and
|
|
// keep going, write what we can.
|
|
}
|
|
|
|
writeStart := (b.Beg + b.Readable) % b.N
|
|
|
|
upperLim := intMin(writeStart+writeCapacity, b.N)
|
|
|
|
k := copy(b.A[b.Use][writeStart:upperLim], p)
|
|
|
|
n += k
|
|
b.Readable += k
|
|
p = p[k:]
|
|
|
|
// we can fill from b.A[b.Use][0:something] from
|
|
// p's remainder, so loop
|
|
}
|
|
}
|
|
|
|
// WriteTo and ReadFrom avoid intermediate allocation and copies.
|
|
|
|
// WriteTo writes data to w until there's no more data to write
|
|
// or when an error occurs. The return value n is the number of
|
|
// bytes written. Any error encountered during the write is also returned.
|
|
func (b *FixedSizeRingBuf) WriteTo(w io.Writer) (n int64, err error) {
|
|
|
|
if b.Readable == 0 {
|
|
return 0, io.EOF
|
|
}
|
|
|
|
extent := b.Beg + b.Readable
|
|
firstWriteLen := intMin(extent, b.N) - b.Beg
|
|
secondWriteLen := b.Readable - firstWriteLen
|
|
if firstWriteLen > 0 {
|
|
m, e := w.Write(b.A[b.Use][b.Beg:(b.Beg + firstWriteLen)])
|
|
n += int64(m)
|
|
b.Advance(m)
|
|
|
|
if e != nil {
|
|
return n, e
|
|
}
|
|
// all bytes should have been written, by definition of
|
|
// Write method in io.Writer
|
|
if m != firstWriteLen {
|
|
return n, io.ErrShortWrite
|
|
}
|
|
}
|
|
if secondWriteLen > 0 {
|
|
m, e := w.Write(b.A[b.Use][0:secondWriteLen])
|
|
n += int64(m)
|
|
b.Advance(m)
|
|
|
|
if e != nil {
|
|
return n, e
|
|
}
|
|
// all bytes should have been written, by definition of
|
|
// Write method in io.Writer
|
|
if m != secondWriteLen {
|
|
return n, io.ErrShortWrite
|
|
}
|
|
}
|
|
|
|
return n, nil
|
|
}
|
|
|
|
// ReadFrom() reads data from r until EOF or error. The return value n
|
|
// is the number of bytes read. Any error except io.EOF encountered
|
|
// during the read is also returned.
|
|
func (b *FixedSizeRingBuf) ReadFrom(r io.Reader) (n int64, err error) {
|
|
for {
|
|
writeCapacity := b.N - b.Readable
|
|
if writeCapacity <= 0 {
|
|
// we are all full
|
|
return n, nil
|
|
}
|
|
writeStart := (b.Beg + b.Readable) % b.N
|
|
upperLim := intMin(writeStart+writeCapacity, b.N)
|
|
|
|
m, e := r.Read(b.A[b.Use][writeStart:upperLim])
|
|
n += int64(m)
|
|
b.Readable += m
|
|
if e == io.EOF {
|
|
return n, nil
|
|
}
|
|
if e != nil {
|
|
return n, e
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *FixedSizeRingBuf) Reset() {
|
|
b.Beg = 0
|
|
b.Readable = 0
|
|
b.Use = 0
|
|
}
|
|
|
|
// Advance(): non-standard, but better than Next(),
|
|
// because we don't have to unwrap our buffer and pay the cpu time
|
|
// for the copy that unwrapping may need.
|
|
// Useful in conjuction/after ReadWithoutAdvance() above.
|
|
func (b *FixedSizeRingBuf) Advance(n int) {
|
|
if n <= 0 {
|
|
return
|
|
}
|
|
if n > b.Readable {
|
|
n = b.Readable
|
|
}
|
|
b.Readable -= n
|
|
b.Beg = (b.Beg + n) % b.N
|
|
}
|
|
|
|
// Adopt(): non-standard.
|
|
//
|
|
// For efficiency's sake, (possibly) take ownership of
|
|
// already allocated slice offered in me.
|
|
//
|
|
// If me is large we will adopt it, and we will potentially then
|
|
// write to the me buffer.
|
|
// If we already have a bigger buffer, copy me into the existing
|
|
// buffer instead.
|
|
func (b *FixedSizeRingBuf) Adopt(me []byte) {
|
|
n := len(me)
|
|
if n > b.N {
|
|
b.A[0] = me
|
|
b.OneMade = false
|
|
b.N = n
|
|
b.Use = 0
|
|
b.Beg = 0
|
|
b.Readable = n
|
|
} else {
|
|
// we already have a larger buffer, reuse it.
|
|
copy(b.A[0], me)
|
|
b.Use = 0
|
|
b.Beg = 0
|
|
b.Readable = n
|
|
}
|
|
}
|
|
|
|
func intMax(a, b int) int {
|
|
if a > b {
|
|
return a
|
|
} else {
|
|
return b
|
|
}
|
|
}
|
|
|
|
func intMin(a, b int) int {
|
|
if a < b {
|
|
return a
|
|
} else {
|
|
return b
|
|
}
|
|
}
|
|
|
|
// Get the (beg, end] indices of the tailing empty buffer of bytes slice that from that is free for writing.
|
|
// Note: not guaranteed to be zeroed. At all.
|
|
func (b *FixedSizeRingBuf) GetEndmostWritable() (beg int, end int) {
|
|
extent := b.Beg + b.Readable
|
|
if extent < b.N {
|
|
return extent, b.N
|
|
}
|
|
|
|
return extent % b.N, b.Beg
|
|
}
|
|
|
|
// Note: not guaranteed to be zeroed.
|
|
func (b *FixedSizeRingBuf) GetEndmostWritableSlice() []byte {
|
|
beg, e := b.GetEndmostWritable()
|
|
return b.A[b.Use][beg:e]
|
|
}
|