Merge pull request #349 from filimonic/settings-separate-file

move Settings and Viewed to separate json files

server/settings/db.go

@@ -15,7 +15,7 @@ type TDB struct {
 	db   *bolt.DB
 }
 
-func NewTDB() *TDB {
+func NewTDB() TorrServerDB {
 	db, err := bolt.Open(filepath.Join(Path, "config.db"), 0o666, &bolt.Options{Timeout: 5 * time.Second})
 	if err != nil {
 		log.TLogln(err)

server/settings/dbreadcache.go

@@ -0,0 +1,60 @@
+package settings
+
+type DBReadCache struct {
+	db        TorrServerDB
+	listCache map[string][]string
+	dataCache map[[2]string][]byte
+}
+
+func NewDBReadCache(db TorrServerDB) TorrServerDB {
+	cdb := &DBReadCache{
+		db:        db,
+		listCache: map[string][]string{},
+		dataCache: map[[2]string][]byte{},
+	}
+	return cdb
+}
+
+func (v *DBReadCache) CloseDB() {
+	v.db.CloseDB()
+	v.db = nil
+	v.listCache = nil
+	v.dataCache = nil
+}
+
+func (v *DBReadCache) Get(xPath, name string) []byte {
+	cacheKey := v.makeDataCacheKey(xPath, name)
+	if data, ok := v.dataCache[cacheKey]; ok {
+		return data
+	}
+	data := v.db.Get(xPath, name)
+	v.dataCache[cacheKey] = data
+	return data
+}
+
+func (v *DBReadCache) Set(xPath, name string, value []byte) {
+	cacheKey := v.makeDataCacheKey(xPath, name)
+	v.dataCache[cacheKey] = value
+	delete(v.listCache, xPath)
+	v.db.Set(xPath, name, value)
+}
+
+func (v *DBReadCache) List(xPath string) []string {
+	if names, ok := v.listCache[xPath]; ok {
+		return names
+	}
+	names := v.db.List(xPath)
+	v.listCache[xPath] = names
+	return names
+}
+
+func (v *DBReadCache) Rem(xPath, name string) {
+	cacheKey := v.makeDataCacheKey(xPath, name)
+	delete(v.dataCache, cacheKey)
+	delete(v.listCache, xPath)
+	v.db.Rem(xPath, name)
+}
+
+func (v *DBReadCache) makeDataCacheKey(xPath, name string) [2]string {
+	return [2]string{xPath, name}
+}

server/settings/jsondb.go

@@ -0,0 +1,159 @@
+package settings
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"server/log"
+	"strings"
+	"sync"
+)
+
+type JsonDB struct {
+	Path              string
+	filenameDelimiter string
+	filenameExtension string
+	fileMode          fs.FileMode
+	xPathDelimeter    string
+}
+
+var jsonDbLocks = make(map[string]*sync.Mutex)
+
+func NewJsonDB() TorrServerDB {
+	settingsDB := &JsonDB{
+		Path:              Path,
+		filenameDelimiter: ".",
+		filenameExtension: ".json",
+		fileMode:          fs.FileMode(0o666),
+		xPathDelimeter:    "/",
+	}
+	return settingsDB
+}
+
+func (v *JsonDB) CloseDB() {
+	// Not necessary
+}
+
+func (v *JsonDB) Set(xPath, name string, value []byte) {
+	var err error = nil
+	jsonObj := map[string]interface{}{}
+
+	if err := json.Unmarshal(value, &jsonObj); err == nil {
+		if filename, err := v.xPathToFilename(xPath); err == nil {
+			v.lock(filename)
+			defer v.unlock(filename)
+			if root, err := v.readJsonFileAsMap(filename); err == nil {
+				root[name] = jsonObj
+				if err = v.writeMapAsJsonFile(filename, root); err == nil {
+					return
+				}
+			}
+		}
+	}
+	v.log(fmt.Sprintf("Set: error writing entry %s->%s", xPath, name), err)
+}
+
+func (v *JsonDB) Get(xPath, name string) []byte {
+	var err error = nil
+	if filename, err := v.xPathToFilename(xPath); err == nil {
+		v.lock(filename)
+		defer v.unlock(filename)
+		if root, err := v.readJsonFileAsMap(filename); err == nil {
+			if jsonData, ok := root[name]; ok {
+				if byteData, err := json.Marshal(jsonData); err == nil {
+					return byteData
+				}
+			} else {
+				// We assume this is not 'error' but 'no entry' which is normal
+				return nil
+			}
+		}
+	}
+	v.log(fmt.Sprintf("Get: error reading entry %s->%s", xPath, name), err)
+	return nil
+}
+
+func (v *JsonDB) List(xPath string) []string {
+	var err error = nil
+	if filename, err := v.xPathToFilename(xPath); err == nil {
+		v.lock(filename)
+		defer v.unlock(filename)
+		if root, err := v.readJsonFileAsMap(filename); err == nil {
+			nameList := make([]string, 0, len(root))
+			for k := range root {
+				nameList = append(nameList, k)
+			}
+			return nameList
+		}
+	}
+	v.log(fmt.Sprintf("List: error reading entries in xPath %s", xPath), err)
+	return nil
+}
+
+func (v *JsonDB) Rem(xPath, name string) {
+	var err error = nil
+	if filename, err := v.xPathToFilename(xPath); err == nil {
+		v.lock(filename)
+		defer v.unlock(filename)
+		if root, err := v.readJsonFileAsMap(filename); err == nil {
+			delete(root, name)
+			v.writeMapAsJsonFile(filename, root)
+			return
+		}
+	}
+	v.log(fmt.Sprintf("Rem: error removing entry %s->%s", xPath, name), err)
+}
+
+func (v *JsonDB) lock(filename string) {
+	var mtx sync.Mutex
+	if mtx, ok := jsonDbLocks[filename]; !ok {
+		mtx = new(sync.Mutex)
+		jsonDbLocks[v.Path] = mtx
+	}
+	mtx.Lock()
+}
+
+func (v *JsonDB) unlock(filename string) {
+	if mtx, ok := jsonDbLocks[filename]; ok {
+		mtx.Unlock()
+	}
+}
+
+func (v *JsonDB) xPathToFilename(xPath string) (string, error) {
+	if pathComponents := strings.Split(xPath, v.xPathDelimeter); len(pathComponents) > 0 {
+		return strings.ToLower(strings.Join(pathComponents, v.filenameDelimiter) + v.filenameExtension), nil
+	}
+	return "", errors.New("xPath has no components")
+}
+
+func (v *JsonDB) readJsonFileAsMap(filename string) (map[string]interface{}, error) {
+	var err error = nil
+	jsonData := map[string]interface{}{}
+	path := filepath.Join(v.Path, filename)
+	if fileData, err := os.ReadFile(path); err == nil {
+		err = json.Unmarshal(fileData, &jsonData)
+	}
+	return jsonData, err
+}
+
+func (v *JsonDB) writeMapAsJsonFile(filename string, o map[string]interface{}) error {
+	var err error = nil
+	path := filepath.Join(v.Path, filename)
+
+	if fileData, err := json.MarshalIndent(o, "", "  "); err == nil {
+		err = os.WriteFile(path, fileData, v.fileMode)
+	}
+	return err
+}
+
+func (v *JsonDB) log(s string, params ...interface{}) {
+	if len(params) > 0 {
+		log.TLogln(fmt.Sprintf("JsonDB: %s: %s", s, fmt.Sprint(params...)))
+	} else {
+		log.TLogln(fmt.Sprintf("JsonDB: %s", s))
+	}
+
+}

server/settings/migrate.go

@@ -2,13 +2,16 @@ package settings
 
 import (
 	"encoding/binary"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
-
-	bolt "go.etcd.io/bbolt"
+	"reflect"
 	"server/log"
 	"server/web/api/utils"
+
+	bolt "go.etcd.io/bbolt"
 )
 
 var dbTorrentsName = []byte("Torrents")
@@ -22,7 +25,8 @@ type torrentOldDB struct {
 	Timestamp int64
 }
 
-func Migrate() {
+// Migrate from torrserver.db to config.db
+func Migrate1() {
 	if _, err := os.Lstat(filepath.Join(Path, "torrserver.db")); os.IsNotExist(err) {
 		return
 	}
@@ -100,3 +104,87 @@ func Migrate() {
 func b2i(v []byte) int64 {
 	return int64(binary.BigEndian.Uint64(v))
 }
+
+/*
+	=== Migrate 2 ===
+
+Migrate 'Settings' and 'Viewed' buckets from BBolt ('config.db')
+to separate JSON files ('settings.json' and 'viewed.json')
+
+'Torrents' data continues to remain in the BBolt database ('config.db')
+due to the fact that BLOBs are stored there
+
+To make user be able to roll settings back, no data is deleted from 'config.db' file.
+*/
+func Migrate2(bboltDB, jsonDB TorrServerDB) error {
+	var err error = nil
+
+	const XPATH_SETTINGS = "Settings"
+	const NAME_BITTORR = "BitTorr"
+	const XPATH_VIEWED = "Viewed"
+
+	if BTsets != nil {
+		msg := "Migrate0002 MUST be called before initializing BTSets"
+		log.TLogln(msg)
+		os.Exit(1)
+	}
+
+	isByteArraysEqualJson := func(a, b []byte) (bool, error) {
+		var objectA interface{}
+		var objectB interface{}
+		var err error = nil
+		if err = json.Unmarshal(a, &objectA); err == nil {
+			if err = json.Unmarshal(b, &objectB); err == nil {
+				return reflect.DeepEqual(objectA, objectB), nil
+			} else {
+				err = fmt.Errorf("Error unmashalling B: %s", err.Error())
+			}
+		} else {
+			err = fmt.Errorf("Error unmashalling A: %s", err.Error())
+		}
+		return false, err
+	}
+
+	migrateXPath := func(xPath, name string) error {
+		if jsonDB.Get(xPath, name) == nil {
+			bboltDBBlob := bboltDB.Get(xPath, name)
+			if bboltDBBlob != nil {
+				log.TLogln(fmt.Sprintf("Attempting to migrate %s->%s from TDB to JsonDB", xPath, name))
+				jsonDB.Set(xPath, name, bboltDBBlob)
+				jsonDBBlob := jsonDB.Get(xPath, name)
+				if isEqual, err := isByteArraysEqualJson(bboltDBBlob, jsonDBBlob); err == nil {
+					if isEqual {
+						log.TLogln(fmt.Sprintf("Migrated %s->%s successful", xPath, name))
+					} else {
+						msg := fmt.Sprintf("Failed to migrate %s->%s TDB to JsonDB: equality check failed", xPath, name)
+						log.TLogln(msg)
+						return errors.New(msg)
+					}
+				} else {
+					msg := fmt.Sprintf("Failed to migrate %s->%s TDB to JsonDB: %s", xPath, name, err)
+					log.TLogln(msg)
+					return errors.New(msg)
+				}
+			}
+		}
+		return nil
+	}
+
+	if err = migrateXPath(XPATH_SETTINGS, NAME_BITTORR); err != nil {
+		return err
+	}
+
+	jsonDBViewedNames := jsonDB.List(XPATH_VIEWED)
+	if len(jsonDBViewedNames) <= 0 {
+		bboltDBViewedNames := bboltDB.List(XPATH_VIEWED)
+		if len(bboltDBViewedNames) > 0 {
+			for _, name := range bboltDBViewedNames {
+				err = migrateXPath(XPATH_VIEWED, name)
+				if err != nil {
+					break
+				}
+			}
+		}
+	}
+	return err
+}

server/settings/settings.go

@@ -1,6 +1,7 @@
 package settings
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -8,7 +9,7 @@ import (
 )
 
 var (
-	tdb      *TDB
+	tdb      TorrServerDB
 	Path     string
 	Port     string
 	Ssl      bool
@@ -24,13 +25,35 @@ var (
 func InitSets(readOnly, searchWA bool) {
 	ReadOnly = readOnly
 	SearchWA = searchWA
-	tdb = NewTDB()
-	if tdb == nil {
-		log.TLogln("Error open db:", filepath.Join(Path, "config.db"))
+
+	bboltDB := NewTDB()
+	if bboltDB == nil {
+		log.TLogln("Error open bboltDB:", filepath.Join(Path, "config.db"))
+		os.Exit(1)
+	}
+
+	jsonDB := NewJsonDB()
+	if jsonDB == nil {
+		log.TLogln("Error open jsonDB")
+		os.Exit(1)
+	}
+
+	dbRouter := NewXPathDBRouter()
+	// First registered DB becomes default route
+	dbRouter.RegisterRoute(jsonDB, "Settings")
+	dbRouter.RegisterRoute(jsonDB, "Viewed")
+	dbRouter.RegisterRoute(bboltDB, "Torrents")
+
+	tdb = NewDBReadCache(dbRouter)
+
+	// We migrate settings here, it must be done before loadBTSets()
+	if err := Migrate2(bboltDB, jsonDB); err != nil {
+		log.TLogln(fmt.Sprintf("Migrate2 failed"))
 		os.Exit(1)
 	}
 	loadBTSets()
-	Migrate()
+	Migrate1()
+
 }
 
 func CloseDB() {

server/settings/torrserverdb.go

@@ -0,0 +1,9 @@
+package settings
+
+type TorrServerDB interface {
+	CloseDB()
+	Get(xPath, name string) []byte
+	Set(xPath, name string, value []byte)
+	List(xPath string) []string
+	Rem(xPath, name string)
+}

server/settings/xpathdbrouter.go

@@ -0,0 +1,117 @@
+package settings
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"server/log"
+	"slices"
+	"sort"
+	"strings"
+)
+
+type XPathDBRouter struct {
+	dbs      []TorrServerDB
+	routes   []string
+	route2db map[string]TorrServerDB
+	dbNames  map[TorrServerDB]string
+}
+
+func NewXPathDBRouter() *XPathDBRouter {
+	router := &XPathDBRouter{
+		dbs:      []TorrServerDB{},
+		dbNames:  map[TorrServerDB]string{},
+		routes:   []string{},
+		route2db: map[string]TorrServerDB{},
+	}
+	return router
+}
+
+func (v *XPathDBRouter) RegisterRoute(db TorrServerDB, xPath string) error {
+	newRoute := v.xPathToRoute(xPath)
+
+	if slices.Contains(v.routes, newRoute) {
+		return errors.New(fmt.Sprintf("route \"%s\" already in routing table", newRoute))
+	}
+
+	// First DB becomes Default DB with default route
+	if len(v.dbs) == 0 && len(newRoute) != 0 {
+		v.RegisterRoute(db, "")
+	}
+
+	if !slices.Contains(v.dbs, db) {
+		v.dbs = append(v.dbs, db)
+		v.dbNames[db] = reflect.TypeOf(db).Elem().Name()
+		v.log(fmt.Sprintf("Registered new DB \"%s\", total %d DBs registered", v.getDBName(db), len(v.dbs)))
+	}
+
+	v.route2db[newRoute] = db
+	v.routes = append(v.routes, newRoute)
+
+	// Sort routes by length descending.
+	//   It is important later to help selecting
+	//   most suitable route in getDBForXPath(xPath)
+	sort.Slice(v.routes, func(iLeft, iRight int) bool {
+		return len(v.routes[iLeft]) > len(v.routes[iRight])
+	})
+	v.log(fmt.Sprintf("Registered new route \"%s\" for DB \"%s\", total %d routes", newRoute, v.getDBName(db), len(v.routes)))
+	return nil
+}
+
+func (v *XPathDBRouter) xPathToRoute(xPath string) string {
+	return strings.ToLower(strings.TrimSpace(xPath))
+}
+
+func (v *XPathDBRouter) getDBForXPath(xPath string) TorrServerDB {
+	if len(v.dbs) == 0 {
+		return nil
+	}
+	lookup_route := v.xPathToRoute(xPath)
+	var db TorrServerDB = nil
+	// Expected v.routes sorted by length descending
+	for _, route_prefix := range v.routes {
+		if strings.HasPrefix(lookup_route, route_prefix) {
+			db = v.route2db[route_prefix]
+			break
+		}
+	}
+	return db
+}
+
+func (v *XPathDBRouter) Get(xPath, name string) []byte {
+	return v.getDBForXPath(xPath).Get(xPath, name)
+}
+
+func (v *XPathDBRouter) Set(xPath, name string, value []byte) {
+	v.getDBForXPath(xPath).Set(xPath, name, value)
+}
+
+func (v *XPathDBRouter) List(xPath string) []string {
+	return v.getDBForXPath(xPath).List(xPath)
+}
+
+func (v *XPathDBRouter) Rem(xPath, name string) {
+	v.getDBForXPath(xPath).Rem(xPath, name)
+}
+
+func (v *XPathDBRouter) CloseDB() {
+	for _, db := range v.dbs {
+		db.CloseDB()
+	}
+	v.dbs = nil
+	v.routes = nil
+	v.route2db = nil
+	v.dbNames = nil
+}
+
+func (v *XPathDBRouter) getDBName(db TorrServerDB) string {
+	return v.dbNames[db]
+}
+
+func (v *XPathDBRouter) log(s string, params ...interface{}) {
+	if len(params) > 0 {
+		log.TLogln(fmt.Sprintf("XPathDBRouter: %s: %s", s, fmt.Sprint(params...)))
+	} else {
+		log.TLogln(fmt.Sprintf("XPathDBRouter: %s", s))
+	}
+}