merge maser

server/cmd/main.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
@@ -23,14 +24,14 @@ import (
 
 type args struct {
 	Port        string `arg:"-p" help:"web server port"`
-	Path        string `arg:"-d" help:"database path"`
-	LogPath     string `arg:"-l" help:"log path"`
-	WebLogPath  string `arg:"-w" help:"web log path"`
+	Path        string `arg:"-d" help:"database dir path"`
+	LogPath     string `arg:"-l" help:"server log file path"`
+	WebLogPath  string `arg:"-w" help:"web access log file path"`
 	RDB         bool   `arg:"-r" help:"start in read-only DB mode"`
-	HttpAuth    bool   `arg:"-a" help:"http auth on all requests"`
-	DontKill    bool   `arg:"-k" help:"dont kill server on signal"`
-	UI          bool   `arg:"-u" help:"run page torrserver in browser"`
-	TorrentsDir string `arg:"-t" help:"autoload torrent from dir"`
+	HttpAuth    bool   `arg:"-a" help:"enable http auth on all requests"`
+	DontKill    bool   `arg:"-k" help:"don't kill server on signal"`
+	UI          bool   `arg:"-u" help:"open torrserver page in browser"`
+	TorrentsDir string `arg:"-t" help:"autoload torrents from dir"`
 }
 
 func (args) Version() string {
@@ -53,6 +54,11 @@ func main() {
 	settings.Path = params.Path
 	settings.HttpAuth = params.HttpAuth
 	log.Init(params.LogPath, params.WebLogPath)
+	fmt.Println("=========== START ===========")
+	fmt.Println("Build Go version:", runtime.Version())
+	if params.HttpAuth {
+		log.TLogln("Use HTTP Auth file", settings.Path+"/accs.db")
+	}
 
 	dnsResolve()
 	Preconfig(params.DontKill)
@@ -78,7 +84,7 @@ func main() {
 func dnsResolve() {
 	addrs, err := net.LookupHost("www.google.com")
 	if len(addrs) == 0 {
-		fmt.Println("Check dns", addrs, err)
+		log.TLogln("Check dns failed", addrs, err)
 
 		fn := func(ctx context.Context, network, address string) (net.Conn, error) {
 			d := net.Dialer{}
@@ -89,8 +95,10 @@ func dnsResolve() {
 			Dial: fn,
 		}
 
-		addrs, err = net.LookupHost("www.themoviedb.org")
-		fmt.Println("Check new dns", addrs, err)
+		addrs, err = net.LookupHost("www.google.com")
+		log.TLogln("Check cloudflare dns", addrs, err)
+	} else {
+		log.TLogln("Check dns OK", addrs, err)
 	}
 }
 

server/cmd/preconfig_pos.go

@@ -3,10 +3,11 @@
 package main
 
 import (
-	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
+		
+	"server/log"
 )
 
 func Preconfig(dkill bool) {
@@ -15,15 +16,14 @@ func Preconfig(dkill bool) {
 		signal.Notify(sigc,
 			syscall.SIGHUP,
 			syscall.SIGINT,
-
 			syscall.SIGPIPE,
 			syscall.SIGTERM,
 			syscall.SIGQUIT)
 		go func() {
 			for s := range sigc {
 				if dkill {
-					fmt.Println("Signal catched:", s)
-					fmt.Println("For stop server, close in api")
+					log.TLogln("Signal catched:", s)
+					log.TLogln("To stop server, close it from web / api")
 				}
 			}
 		}()

server/go.mod

@@ -5,7 +5,7 @@ replace (
 	github.com/anacrolix/torrent v1.2.6 => github.com/yourok/torrent v0.0.0-20210427182412-1ef78c0fb02c
 )
 
-go 1.16
+go 1.17
 
 require (
 	github.com/alexflint/go-arg v1.3.0
@@ -18,4 +18,3 @@ require (
 	github.com/pkg/errors v0.8.1
 	go.etcd.io/bbolt v1.3.5
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
-)

server/log/log.go

@@ -35,7 +35,7 @@ func Init(path, webpath string) {
 
 	if path != "" {
 		if fi, err := os.Lstat(path); err == nil {
-			if fi.Size() >= 1*1024*1024*1024 {
+			if fi.Size() >= 100*1024*1024 { // 100MB
 				os.Remove(path)
 			}
 		}
@@ -47,6 +47,10 @@ func Init(path, webpath string) {
 		logFile = ff
 		os.Stdout = ff
 		os.Stderr = ff
+		//https://stackoverflow.com/a/36140590
+		//fmt.Print(time.Now().UTC().Format("2006-01-02T15:04:05.999Z") + " TLOG  " + string(bytes))
+		log.SetFlags(log.Ldate|log.Ltime|log.LUTC|log.Lmsgprefix)
+		log.SetPrefix("0000 TLOG  ")
 		log.SetOutput(ff)
 	}
 }

server/server.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
 
@@ -15,8 +16,18 @@ func Start(port string, roSets bool) {
 	if port == "" {
 		port = "8090"
 	}
-	go cleanCache()
-	web.Start(port)
+	log.TLogln("Check web port", port)
+	l, err := net.Listen("tcp", ":"+port)
+	if l != nil {
+		l.Close()
+	}
+	if err != nil {
+		log.TLogln("Port", port, "already in use! Abort")
+		os.Exit(1)
+	} else {
+	  go cleanCache()
+	  web.Start(port)
+	}
 }
 
 func cleanCache() {

server/settings/btsets.go

@@ -40,7 +40,6 @@ type BTSets struct {
 	ConnectionsLimit  int
 	DhtConnectionLimit int // 0 - inf
 	PeersListenPort int
-	//Strategy           int // 0 - RequestStrategyDuplicateRequestTimeout, 1 - RequestStrategyFuzzing, 2 - RequestStrategyFastest
 }
 
 func (v *BTSets) String() string {

server/torr/apihelper.go

@@ -209,6 +209,7 @@ func SetDefSettings() {
 func Shutdown() {
 	bts.Disconnect()
 	sets.CloseDB()
+	log.TLogln("Received shutdown. Quit")
 	os.Exit(0)
 }
 

server/torr/btserver.go

@@ -61,6 +61,7 @@ func (bt *BTServer) configure() {
 
 	userAgent := "qBittorrent/4.3.2"
 	peerID := "-qB4320-"
+	upnpID := "TorrServer"
 	cliVers := userAgent //"uTorrent/2210(25302)"
 
 	bt.config.Debug = settings.BTsets.EnableDebug
@@ -74,6 +75,7 @@ func (bt *BTServer) configure() {
 	bt.config.IPBlocklist = blocklist
 	bt.config.Bep20 = peerID
 	bt.config.PeerID = utils.PeerIDRandom(peerID)
+	bt.config.UpnpID = upnpID
 	bt.config.HTTPUserAgent = userAgent
 	bt.config.ExtendedHandshakeClientVersion = cliVers
 	bt.config.EstablishedConnsPerTorrent = settings.BTsets.ConnectionsLimit
@@ -93,11 +95,12 @@ func (bt *BTServer) configure() {
 		bt.config.UploadRateLimiter = utils.Limit(settings.BTsets.UploadRateLimit * 1024)
 	}
 	if settings.BTsets.PeersListenPort > 0 {
+		log.Println("Set listen port", settings.BTsets.PeersListenPort)
 		bt.config.ListenPort = settings.BTsets.PeersListenPort
 	} else {
-		log.Println("Find upnp port")
 		upnpport := 32000
 		for {
+			log.Println("Check upnp port", upnpport)
 			l, err := net.Listen("tcp", ":"+strconv.Itoa(upnpport))
 			if l != nil {
 				l.Close()
@@ -111,7 +114,7 @@ func (bt *BTServer) configure() {
 		bt.config.ListenPort = upnpport
 	}
 
-	log.Println("Configure client:", settings.BTsets)
+	log.Println("Client config:", settings.BTsets)
 }
 
 func (bt *BTServer) GetTorrent(hash torrent.InfoHash) *Torrent {

server/torr/preload.go

@@ -63,7 +63,11 @@ func (t *Torrent) Preload(index int, size int64) {
 			}
 		}()
 
-		mb5 := int64(5 * 1024 * 1024)
+		// mb5 -> 8/16 MB
+		mb5 := int64(t.Info().PieceLength)
+		if (mb5 < 8 * 1024 * 1024) {
+			mb5 = 8 * 1024 * 1024
+		}
 
 		readerStart := file.NewReader()
 		defer readerStart.Close()

server/torr/storage/state/state.go

@@ -20,6 +20,7 @@ type ItemState struct {
 	Length    int64
 	Size      int64
 	Completed bool
+	Priority  int
 }
 
 type ReaderState struct {

server/torr/storage/torrstor/cache.go

@@ -35,12 +35,11 @@ type Cache struct {
 	muReaders sync.Mutex
 
 	isRemove bool
+	isClosed bool
 	muRemove sync.Mutex
 	torrent  *torrent.Torrent
 }
 
-const FileRangeNotDelete = 5 * 1024 * 1024
-
 func NewCache(capacity int64, storage *Storage) *Cache {
 	ret := &Cache{
 		capacity: capacity,
@@ -89,6 +88,8 @@ func (c *Cache) Piece(m metainfo.Piece) storage.PieceImpl {
 
 func (c *Cache) Close() error {
 	log.TLogln("Close cache for:", c.hash)
+	c.isClosed = true
+
 	delete(c.storage.caches, c.hash)
 
 	if settings.BTsets.RemoveCacheOnDrop {
@@ -114,14 +115,16 @@ func (c *Cache) Close() error {
 }
 
 func (c *Cache) removePiece(piece *Piece) {
-	piece.Release()
+	if !c.isClosed {
+		piece.Release()
+	}
 }
 
 func (c *Cache) AdjustRA(readahead int64) {
 	if settings.BTsets.CacheSize == 0 {
 		c.capacity = readahead * 3
 	}
-	if len(c.readers) > 0 {
+	if c.Readers() > 0 {
 		c.muReaders.Lock()
 		for r, _ := range c.readers {
 			r.SetReadahead(readahead)
@@ -145,14 +148,16 @@ func (c *Cache) GetState() *state.CacheState {
 					Size:      p.Size,
 					Length:    c.pieceLength,
 					Completed: p.Complete,
+					Priority:  int(c.torrent.PieceState(p.Id).Priority),
 				}
 			}
 		}
 	}
 
 	readersState := make([]*state.ReaderState, 0)
-	c.muReaders.Lock()
-	if len(c.readers) > 0 {
+	
+	if c.Readers() > 0 {
+		c.muReaders.Lock()
 		for r, _ := range c.readers {
 			rng := r.getPiecesRange()
 			pc := r.getReaderPiece()
@@ -162,8 +167,8 @@ func (c *Cache) GetState() *state.CacheState {
 				Reader: pc,
 			})
 		}
+		c.muReaders.Unlock()
 	}
-	c.muReaders.Unlock()
 
 	c.filled = fill
 	cState.Capacity = c.capacity
@@ -177,7 +182,7 @@ func (c *Cache) GetState() *state.CacheState {
 }
 
 func (c *Cache) cleanPieces() {
-	if c.isRemove {
+	if c.isRemove || c.isClosed {
 		return
 	}
 	c.muRemove.Lock()
@@ -236,24 +241,37 @@ func (c *Cache) getRemPieces() []*Piece {
 
 	c.updatePriority()
 
+	c.muReaders.Lock()
 	for r, _ := range c.readers {
 		if c.isIdInFileBE(ranges, r.getReaderPiece()) {
 			continue
 		}
-		pc := r.getReaderPiece()
+		readerPos := r.getReaderPiece()
+		readerRAHPos := r.getReaderRAHPiece()
 		end := r.getPiecesRange().End
-		limit := 5
-
-		for pc <= end && limit > 0 {
-			if !c.pieces[pc].Complete {
-				if c.torrent.PieceState(pc).Priority == torrent.PiecePriorityNone {
-					c.torrent.Piece(pc).SetPriority(torrent.PiecePriorityNormal)
+		count := int(16 * 1024 * 1024 / c.pieceLength * 5) // 80 MB
+		if count > 40 {
+			count = 40
+		}
+		limit := 0
+		for i := readerPos; i < end && limit < count; i++ {
+			if !c.pieces[i].Complete {
+				if i == readerPos {
+					c.torrent.Piece(i).SetPriority(torrent.PiecePriorityNow)
+				} else if i == readerPos + 1 {
+					c.torrent.Piece(i).SetPriority(torrent.PiecePriorityNext)
+				} else if i > readerPos && i <= readerRAHPos {
+					c.torrent.Piece(i).SetPriority(torrent.PiecePriorityReadahead)
+				} else if i > readerRAHPos && i <= readerPos + (end - readerPos)/2 && c.torrent.PieceState(i).Priority != torrent.PiecePriorityHigh {
+					c.torrent.Piece(i).SetPriority(torrent.PiecePriorityHigh)
+				} else if i > readerPos + (end - readerPos)/2 && c.torrent.PieceState(i).Priority != torrent.PiecePriorityNormal {
+					c.torrent.Piece(i).SetPriority(torrent.PiecePriorityNormal)
 				}
-				limit--
+				limit++
 			}
-			pc++
 		}
 	}
+	c.muReaders.Unlock()
 
 	sort.Slice(piecesRemove, func(i, j int) bool {
 		return piecesRemove[i].Accessed < piecesRemove[j].Accessed
@@ -264,14 +282,21 @@ func (c *Cache) getRemPieces() []*Piece {
 }
 
 func (c *Cache) isIdInFileBE(ranges []Range, id int) bool {
+	
+	// keep 8/16 MB
+	FileRangeNotDelete := int64(c.pieceLength)
+	if (FileRangeNotDelete < 8 * 1024 * 1024) {
+		FileRangeNotDelete = 8 * 1024 * 1024
+	}
+
 	for _, rng := range ranges {
 		ss := int(rng.File.Offset() / c.pieceLength)
-		se := int((FileRangeNotDelete + rng.File.Offset()) / c.pieceLength)
+		se := int((rng.File.Offset() + FileRangeNotDelete) / c.pieceLength)
 
 		es := int((rng.File.Offset() + rng.File.Length() - FileRangeNotDelete) / c.pieceLength)
 		ee := int((rng.File.Offset() + rng.File.Length()) / c.pieceLength)
 
-		if id >= ss && id <= se || id >= es && id <= ee {
+		if id >= ss && id < se || id > es && id <= ee {
 			return true
 		}
 	}

server/torr/storage/torrstor/piece.go

@@ -73,5 +73,10 @@ func (p *Piece) Release() {
 	} else {
 		p.dPiece.Release()
 	}
-
+	// TODO: check this merge
+	if !p.cache.isClosed {
+		p.cache.torrent.Piece(p.Id).SetPriority(torrent.PiecePriorityNone)
+		// fix remove pieces hash
+		p.cache.torrent.Piece(p.Id).UpdateCompletion()
+	}
 }

server/torr/storage/torrstor/reader.go

@@ -114,7 +114,7 @@ func (r *Reader) Close() {
 	// this struct close in cache
 	r.isClosed = true
 	if len(r.file.Torrent().Files()) > 0 {
-		r.Reader.Close()
+	 	r.Reader.Close()
 	}
 	go r.cache.getRemPieces()
 }
@@ -125,8 +125,11 @@ func (r *Reader) getPiecesRange() Range {
 }
 
 func (r *Reader) getReaderPiece() int {
-	readerOff := r.offset
-	return r.getPieceNum(readerOff)
+	return r.getPieceNum(r.offset)
+}
+
+func (r *Reader) getReaderRAHPiece() int {
+	return r.getPieceNum(r.offset + r.readahead)
 }
 
 func (r *Reader) getPieceNum(offset int64) int {

server/torr/stream.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"net"
 	"net/http"
 	"time"
 
@@ -46,8 +47,13 @@ func (t *Torrent) Stream(fileID int, req *http.Request, resp http.ResponseWriter
 
 	reader := t.NewReader(file)
 
-	log.Println("Connect client")
-
+	host, port, err := net.SplitHostPort(req.RemoteAddr)
+	if err != nil {
+		log.Println("Connect client")
+	} else {
+		log.Println("Connect client", host, port)
+	}
+	
 	sets.SetViewed(&sets.Viewed{t.Hash().HexString(), fileID})
 
 	resp.Header().Set("Connection", "close")
@@ -56,6 +62,10 @@ func (t *Torrent) Stream(fileID int, req *http.Request, resp http.ResponseWriter
 	http.ServeContent(resp, req, file.Path(), time.Unix(t.Timestamp, 0), reader)
 
 	t.CloseReader(reader)
-	log.Println("Disconnect client")
+	if err != nil {
+		log.Println("Disconnect client")
+	} else {
+		log.Println("Disconnect client", host, port)
+	}
 	return nil
 }

server/version/version.go

@@ -1,3 +1,3 @@
 package version
 
-const Version = "MatriX.102.OE"
+const Version = "MatriX.103.OE"

server/web/api/torrents.go

@@ -71,7 +71,7 @@ func addTorrent(req torrReqJS, c *gin.Context) {
 	req.Link = strings.ReplaceAll(req.Link, "&", "&")
 	torrSpec, err := utils.ParseLink(req.Link)
 	if err != nil {
-		log.TLogln("error add torrent:", err)
+		log.TLogln("error parse link:", err)
 		c.AbortWithError(http.StatusBadRequest, err)
 		return
 	}
@@ -85,12 +85,18 @@ func addTorrent(req torrReqJS, c *gin.Context) {
 
 	go func() {
 		if !tor.GotInfo() {
-			log.TLogln("error add torrent:", "timeout connection torrent")
+			log.TLogln("error add torrent:", "timeout connection get torrent info")
 			return
 		}
 
 		if tor.Title == "" {
-			tor.Title = tor.Name()
+			tor.Title = torrSpec.DisplayName // prefer dn over name
+			tor.Title = strings.ReplaceAll(tor.Title, "rutor.info", "")
+			tor.Title = strings.ReplaceAll(tor.Title, "_", " ")
+			tor.Title = strings.Trim(tor.Title, " ")
+			if tor.Title == "" {
+				tor.Title = tor.Name()
+			}
 		}
 
 		if req.SaveToDB {

server/web/pages/template/pages/static/js/2.97916048.chunk.js.LICENSE.txt

@@ -0,0 +1,92 @@
+/*
+object-assign
+(c) Sindre Sorhus
+@license MIT
+*/
+
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <http://feross.org>
+ * @license  MIT
+ */
+
+/*! blob-to-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/*! https://mths.be/punycode v1.4.1 by @mathias */
+
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/*! magnet-uri. MIT License. WebTorrent LLC <https://webtorrent.io/opensource> */
+
+/*! parse-torrent. MIT License. WebTorrent LLC <https://webtorrent.io/opensource> */
+
+/*! queue-microtask. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/*! simple-concat. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/*! simple-get. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+
+/**
+ * A better abstraction over CSS.
+ *
+ * @copyright Oleg Isonen (Slobodskoi) / Isonen 2014-present
+ * @website https://github.com/cssinjs/jss
+ * @license MIT
+ */
+
+/** @license React v0.20.2
+ * scheduler.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v16.13.1
+ * react-is.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react-dom.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react-is.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v17.0.2
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */

server/web/server.go

@@ -51,7 +51,7 @@ func Start(port string) {
 		api.SetupRoute(&route.RouterGroup)
 		pages.SetupRoute(&route.RouterGroup)
 	}
-	log.TLogln("Start web", port)
+	log.TLogln("Start web server at port", port)
 	waitChan <- route.Run(":" + port)
 }