diff --git a/server/settings/db.go b/server/settings/db.go index 03ec985..01f0b5b 100644 --- a/server/settings/db.go +++ b/server/settings/db.go @@ -15,7 +15,7 @@ type TDB struct { db *bolt.DB } -func NewTDB() *TDB { +func NewTDB() TorrServerDB { db, err := bolt.Open(filepath.Join(Path, "config.db"), 0o666, &bolt.Options{Timeout: 5 * time.Second}) if err != nil { log.TLogln(err) diff --git a/server/settings/dbreadcache.go b/server/settings/dbreadcache.go new file mode 100644 index 0000000..4158a4c --- /dev/null +++ b/server/settings/dbreadcache.go @@ -0,0 +1,60 @@ +package settings + +type DBReadCache struct { + db TorrServerDB + listCache map[string][]string + dataCache map[[2]string][]byte +} + +func NewDBReadCache(db TorrServerDB) TorrServerDB { + cdb := &DBReadCache{ + db: db, + listCache: map[string][]string{}, + dataCache: map[[2]string][]byte{}, + } + return cdb +} + +func (v *DBReadCache) CloseDB() { + v.db.CloseDB() + v.db = nil + v.listCache = nil + v.dataCache = nil +} + +func (v *DBReadCache) Get(xPath, name string) []byte { + cacheKey := v.makeDataCacheKey(xPath, name) + if data, ok := v.dataCache[cacheKey]; ok { + return data + } + data := v.db.Get(xPath, name) + v.dataCache[cacheKey] = data + return data +} + +func (v *DBReadCache) Set(xPath, name string, value []byte) { + cacheKey := v.makeDataCacheKey(xPath, name) + v.dataCache[cacheKey] = value + delete(v.listCache, xPath) + v.db.Set(xPath, name, value) +} + +func (v *DBReadCache) List(xPath string) []string { + if names, ok := v.listCache[xPath]; ok { + return names + } + names := v.db.List(xPath) + v.listCache[xPath] = names + return names +} + +func (v *DBReadCache) Rem(xPath, name string) { + cacheKey := v.makeDataCacheKey(xPath, name) + delete(v.dataCache, cacheKey) + delete(v.listCache, xPath) + v.db.Rem(xPath, name) +} + +func (v *DBReadCache) makeDataCacheKey(xPath, name string) [2]string { + return [2]string{xPath, name} +} diff --git a/server/settings/jsondb.go b/server/settings/jsondb.go new file mode 100644 index 0000000..a4c4180 --- /dev/null +++ b/server/settings/jsondb.go @@ -0,0 +1,159 @@ +package settings + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "server/log" + "strings" + "sync" +) + +type JsonDB struct { + Path string + filenameDelimiter string + filenameExtension string + fileMode fs.FileMode + xPathDelimeter string +} + +var jsonDbLocks = make(map[string]*sync.Mutex) + +func NewJsonDB() TorrServerDB { + settingsDB := &JsonDB{ + Path: Path, + filenameDelimiter: ".", + filenameExtension: ".json", + fileMode: fs.FileMode(0o666), + xPathDelimeter: "/", + } + return settingsDB +} + +func (v *JsonDB) CloseDB() { + // Not necessary +} + +func (v *JsonDB) Set(xPath, name string, value []byte) { + var err error = nil + jsonObj := map[string]interface{}{} + + if err := json.Unmarshal(value, &jsonObj); err == nil { + if filename, err := v.xPathToFilename(xPath); err == nil { + v.lock(filename) + defer v.unlock(filename) + if root, err := v.readJsonFileAsMap(filename); err == nil { + root[name] = jsonObj + if err = v.writeMapAsJsonFile(filename, root); err == nil { + return + } + } + } + } + v.log(fmt.Sprintf("Set: error writing entry %s->%s", xPath, name), err) +} + +func (v *JsonDB) Get(xPath, name string) []byte { + var err error = nil + if filename, err := v.xPathToFilename(xPath); err == nil { + v.lock(filename) + defer v.unlock(filename) + if root, err := v.readJsonFileAsMap(filename); err == nil { + if jsonData, ok := root[name]; ok { + if byteData, err := json.Marshal(jsonData); err == nil { + return byteData + } + } else { + // We assume this is not 'error' but 'no entry' which is normal + return nil + } + } + } + v.log(fmt.Sprintf("Get: error reading entry %s->%s", xPath, name), err) + return nil +} + +func (v *JsonDB) List(xPath string) []string { + var err error = nil + if filename, err := v.xPathToFilename(xPath); err == nil { + v.lock(filename) + defer v.unlock(filename) + if root, err := v.readJsonFileAsMap(filename); err == nil { + nameList := make([]string, 0, len(root)) + for k := range root { + nameList = append(nameList, k) + } + return nameList + } + } + v.log(fmt.Sprintf("List: error reading entries in xPath %s", xPath), err) + return nil +} + +func (v *JsonDB) Rem(xPath, name string) { + var err error = nil + if filename, err := v.xPathToFilename(xPath); err == nil { + v.lock(filename) + defer v.unlock(filename) + if root, err := v.readJsonFileAsMap(filename); err == nil { + delete(root, name) + v.writeMapAsJsonFile(filename, root) + return + } + } + v.log(fmt.Sprintf("Rem: error removing entry %s->%s", xPath, name), err) +} + +func (v *JsonDB) lock(filename string) { + var mtx sync.Mutex + if mtx, ok := jsonDbLocks[filename]; !ok { + mtx = new(sync.Mutex) + jsonDbLocks[v.Path] = mtx + } + mtx.Lock() +} + +func (v *JsonDB) unlock(filename string) { + if mtx, ok := jsonDbLocks[filename]; ok { + mtx.Unlock() + } +} + +func (v *JsonDB) xPathToFilename(xPath string) (string, error) { + if pathComponents := strings.Split(xPath, v.xPathDelimeter); len(pathComponents) > 0 { + return strings.ToLower(strings.Join(pathComponents, v.filenameDelimiter) + v.filenameExtension), nil + } + return "", errors.New("xPath has no components") +} + +func (v *JsonDB) readJsonFileAsMap(filename string) (map[string]interface{}, error) { + var err error = nil + jsonData := map[string]interface{}{} + path := filepath.Join(v.Path, filename) + if fileData, err := os.ReadFile(path); err == nil { + err = json.Unmarshal(fileData, &jsonData) + } + return jsonData, err +} + +func (v *JsonDB) writeMapAsJsonFile(filename string, o map[string]interface{}) error { + var err error = nil + path := filepath.Join(v.Path, filename) + + if fileData, err := json.MarshalIndent(o, "", " "); err == nil { + err = os.WriteFile(path, fileData, v.fileMode) + } + return err +} + +func (v *JsonDB) log(s string, params ...interface{}) { + if len(params) > 0 { + log.TLogln(fmt.Sprintf("JsonDB: %s: %s", s, fmt.Sprint(params...))) + } else { + log.TLogln(fmt.Sprintf("JsonDB: %s", s)) + } + +} diff --git a/server/settings/migrate.go b/server/settings/migrate.go index e0c6ce1..2ae12f7 100644 --- a/server/settings/migrate.go +++ b/server/settings/migrate.go @@ -2,13 +2,16 @@ package settings import ( "encoding/binary" + "encoding/json" + "errors" "fmt" "os" "path/filepath" - - bolt "go.etcd.io/bbolt" + "reflect" "server/log" "server/web/api/utils" + + bolt "go.etcd.io/bbolt" ) var dbTorrentsName = []byte("Torrents") @@ -22,7 +25,8 @@ type torrentOldDB struct { Timestamp int64 } -func Migrate() { +// Migrate from torrserver.db to config.db +func Migrate1() { if _, err := os.Lstat(filepath.Join(Path, "torrserver.db")); os.IsNotExist(err) { return } @@ -100,3 +104,87 @@ func Migrate() { func b2i(v []byte) int64 { return int64(binary.BigEndian.Uint64(v)) } + +/* + === Migrate 2 === + +Migrate 'Settings' and 'Viewed' buckets from BBolt ('config.db') +to separate JSON files ('settings.json' and 'viewed.json') + +'Torrents' data continues to remain in the BBolt database ('config.db') +due to the fact that BLOBs are stored there + +To make user be able to roll settings back, no data is deleted from 'config.db' file. +*/ +func Migrate2(bboltDB, jsonDB TorrServerDB) error { + var err error = nil + + const XPATH_SETTINGS = "Settings" + const NAME_BITTORR = "BitTorr" + const XPATH_VIEWED = "Viewed" + + if BTsets != nil { + msg := "Migrate0002 MUST be called before initializing BTSets" + log.TLogln(msg) + os.Exit(1) + } + + isByteArraysEqualJson := func(a, b []byte) (bool, error) { + var objectA interface{} + var objectB interface{} + var err error = nil + if err = json.Unmarshal(a, &objectA); err == nil { + if err = json.Unmarshal(b, &objectB); err == nil { + return reflect.DeepEqual(objectA, objectB), nil + } else { + err = fmt.Errorf("Error unmashalling B: %s", err.Error()) + } + } else { + err = fmt.Errorf("Error unmashalling A: %s", err.Error()) + } + return false, err + } + + migrateXPath := func(xPath, name string) error { + if jsonDB.Get(xPath, name) == nil { + bboltDBBlob := bboltDB.Get(xPath, name) + if bboltDBBlob != nil { + log.TLogln(fmt.Sprintf("Attempting to migrate %s->%s from TDB to JsonDB", xPath, name)) + jsonDB.Set(xPath, name, bboltDBBlob) + jsonDBBlob := jsonDB.Get(xPath, name) + if isEqual, err := isByteArraysEqualJson(bboltDBBlob, jsonDBBlob); err == nil { + if isEqual { + log.TLogln(fmt.Sprintf("Migrated %s->%s successful", xPath, name)) + } else { + msg := fmt.Sprintf("Failed to migrate %s->%s TDB to JsonDB: equality check failed", xPath, name) + log.TLogln(msg) + return errors.New(msg) + } + } else { + msg := fmt.Sprintf("Failed to migrate %s->%s TDB to JsonDB: %s", xPath, name, err) + log.TLogln(msg) + return errors.New(msg) + } + } + } + return nil + } + + if err = migrateXPath(XPATH_SETTINGS, NAME_BITTORR); err != nil { + return err + } + + jsonDBViewedNames := jsonDB.List(XPATH_VIEWED) + if len(jsonDBViewedNames) <= 0 { + bboltDBViewedNames := bboltDB.List(XPATH_VIEWED) + if len(bboltDBViewedNames) > 0 { + for _, name := range bboltDBViewedNames { + err = migrateXPath(XPATH_VIEWED, name) + if err != nil { + break + } + } + } + } + return err +} diff --git a/server/settings/settings.go b/server/settings/settings.go index 8c5906b..34e9bf9 100644 --- a/server/settings/settings.go +++ b/server/settings/settings.go @@ -1,6 +1,7 @@ package settings import ( + "fmt" "os" "path/filepath" @@ -8,7 +9,7 @@ import ( ) var ( - tdb *TDB + tdb TorrServerDB Path string Port string Ssl bool @@ -24,13 +25,35 @@ var ( func InitSets(readOnly, searchWA bool) { ReadOnly = readOnly SearchWA = searchWA - tdb = NewTDB() - if tdb == nil { - log.TLogln("Error open db:", filepath.Join(Path, "config.db")) + + bboltDB := NewTDB() + if bboltDB == nil { + log.TLogln("Error open bboltDB:", filepath.Join(Path, "config.db")) + os.Exit(1) + } + + jsonDB := NewJsonDB() + if jsonDB == nil { + log.TLogln("Error open jsonDB") + os.Exit(1) + } + + dbRouter := NewXPathDBRouter() + // First registered DB becomes default route + dbRouter.RegisterRoute(jsonDB, "Settings") + dbRouter.RegisterRoute(jsonDB, "Viewed") + dbRouter.RegisterRoute(bboltDB, "Torrents") + + tdb = NewDBReadCache(dbRouter) + + // We migrate settings here, it must be done before loadBTSets() + if err := Migrate2(bboltDB, jsonDB); err != nil { + log.TLogln(fmt.Sprintf("Migrate2 failed")) os.Exit(1) } loadBTSets() - Migrate() + Migrate1() + } func CloseDB() { diff --git a/server/settings/torrserverdb.go b/server/settings/torrserverdb.go new file mode 100644 index 0000000..c9bb50e --- /dev/null +++ b/server/settings/torrserverdb.go @@ -0,0 +1,9 @@ +package settings + +type TorrServerDB interface { + CloseDB() + Get(xPath, name string) []byte + Set(xPath, name string, value []byte) + List(xPath string) []string + Rem(xPath, name string) +} diff --git a/server/settings/xpathdbrouter.go b/server/settings/xpathdbrouter.go new file mode 100644 index 0000000..fb6a07c --- /dev/null +++ b/server/settings/xpathdbrouter.go @@ -0,0 +1,117 @@ +package settings + +import ( + "errors" + "fmt" + "reflect" + "server/log" + "slices" + "sort" + "strings" +) + +type XPathDBRouter struct { + dbs []TorrServerDB + routes []string + route2db map[string]TorrServerDB + dbNames map[TorrServerDB]string +} + +func NewXPathDBRouter() *XPathDBRouter { + router := &XPathDBRouter{ + dbs: []TorrServerDB{}, + dbNames: map[TorrServerDB]string{}, + routes: []string{}, + route2db: map[string]TorrServerDB{}, + } + return router +} + +func (v *XPathDBRouter) RegisterRoute(db TorrServerDB, xPath string) error { + newRoute := v.xPathToRoute(xPath) + + if slices.Contains(v.routes, newRoute) { + return errors.New(fmt.Sprintf("route \"%s\" already in routing table", newRoute)) + } + + // First DB becomes Default DB with default route + if len(v.dbs) == 0 && len(newRoute) != 0 { + v.RegisterRoute(db, "") + } + + if !slices.Contains(v.dbs, db) { + v.dbs = append(v.dbs, db) + v.dbNames[db] = reflect.TypeOf(db).Elem().Name() + v.log(fmt.Sprintf("Registered new DB \"%s\", total %d DBs registered", v.getDBName(db), len(v.dbs))) + } + + v.route2db[newRoute] = db + v.routes = append(v.routes, newRoute) + + // Sort routes by length descending. + // It is important later to help selecting + // most suitable route in getDBForXPath(xPath) + sort.Slice(v.routes, func(iLeft, iRight int) bool { + return len(v.routes[iLeft]) > len(v.routes[iRight]) + }) + v.log(fmt.Sprintf("Registered new route \"%s\" for DB \"%s\", total %d routes", newRoute, v.getDBName(db), len(v.routes))) + return nil +} + +func (v *XPathDBRouter) xPathToRoute(xPath string) string { + return strings.ToLower(strings.TrimSpace(xPath)) +} + +func (v *XPathDBRouter) getDBForXPath(xPath string) TorrServerDB { + if len(v.dbs) == 0 { + return nil + } + lookup_route := v.xPathToRoute(xPath) + var db TorrServerDB = nil + // Expected v.routes sorted by length descending + for _, route_prefix := range v.routes { + if strings.HasPrefix(lookup_route, route_prefix) { + db = v.route2db[route_prefix] + break + } + } + return db +} + +func (v *XPathDBRouter) Get(xPath, name string) []byte { + return v.getDBForXPath(xPath).Get(xPath, name) +} + +func (v *XPathDBRouter) Set(xPath, name string, value []byte) { + v.getDBForXPath(xPath).Set(xPath, name, value) +} + +func (v *XPathDBRouter) List(xPath string) []string { + return v.getDBForXPath(xPath).List(xPath) +} + +func (v *XPathDBRouter) Rem(xPath, name string) { + v.getDBForXPath(xPath).Rem(xPath, name) +} + +func (v *XPathDBRouter) CloseDB() { + for _, db := range v.dbs { + db.CloseDB() + } + v.dbs = nil + v.routes = nil + v.route2db = nil + v.dbNames = nil +} + +func (v *XPathDBRouter) getDBName(db TorrServerDB) string { + return v.dbNames[db] +} + +func (v *XPathDBRouter) log(s string, params ...interface{}) { + if len(params) > 0 { + log.TLogln(fmt.Sprintf("XPathDBRouter: %s: %s", s, fmt.Sprint(params...))) + } else { + log.TLogln(fmt.Sprintf("XPathDBRouter: %s", s)) + } +}