WIP: Persistence

.gitignore

@@ -5,3 +5,5 @@ go-jwlm
 2020-08-06.jwlibrary
 2020-03-30.jwlibrary
 dist
+*.exe
+*.wasm

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec
 	github.com/codeclysm/extract/v3 v3.0.2
 	github.com/davecgh/go-spew v1.1.1
+	github.com/fritzbauer/go-sqlite3-js v0.0.0-20210228162201-a9149e4032a8
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-openapi/errors v0.19.9 // indirect
 	github.com/go-openapi/strfmt v0.19.11 // indirect

go.sum

@@ -61,6 +61,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fritzbauer/go-sqlite3-js v0.0.0-20210227210033-af80406c0f5d h1:P52Q8OHtQfl3wECj7xzG8YIQ/4sieEk2e4fdlQNMpjA=
+github.com/fritzbauer/go-sqlite3-js v0.0.0-20210227210033-af80406c0f5d/go.mod h1:4l4Wgw6/8enHFSGtRO5PAfMwcnPjL3JX8Wtp+N+geCU=
+github.com/fritzbauer/go-sqlite3-js v0.0.0-20210228162201-a9149e4032a8 h1:GpYToYmFKCBFloxgGVrsw/b8CsrvI5x1lhhQ7zc/rKQ=
+github.com/fritzbauer/go-sqlite3-js v0.0.0-20210228162201-a9149e4032a8/go.mod h1:4l4Wgw6/8enHFSGtRO5PAfMwcnPjL3JX8Wtp+N+geCU=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -283,6 +287,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

main.go

@@ -1,3 +1,5 @@
+// +build !js
+
 package main
 
 import "github.com/AndreasSko/go-jwlm/cmd"

main_wasm.go

@@ -0,0 +1,48 @@
+// +build js
+
+package main
+
+import (
+	"fmt"
+	"syscall/js"
+
+	"github.com/AndreasSko/go-jwlm/wasm"
+)
+
+func mergeJs(this js.Value, inputs []js.Value) interface{} {
+	leftDbArr := inputs[0]
+	rightDbArr := inputs[1]
+	mergedDbName := inputs[2].String()
+	//mergedDbArr := make([]uint8, leftDbArr.Get("byteLength").Int())
+
+	leftBuf := make([]uint8, leftDbArr.Get("byteLength").Int())
+	rightBuf := make([]uint8, rightDbArr.Get("byteLength").Int())
+
+	js.CopyBytesToGo(leftBuf, leftDbArr)
+	js.CopyBytesToGo(rightBuf, rightDbArr)
+
+	mergedDb := wasm.Merge(leftBuf, rightBuf, mergedDbName)
+
+	//js.CopyBytesToJS(mergedDbArr, mergedDb)
+	fmt.Printf("Merged. Returning %d bytes\n", len(mergedDb))
+	mergedJsData := js.Global().Get("Uint8Array").New(len(mergedDb))
+	js.CopyBytesToJS(mergedJsData, mergedDb)
+	return mergedJsData
+
+}
+
+func registerCallbacks() {
+	js.Global().Set("mergeJs", js.FuncOf(mergeJs))
+}
+
+func main() {
+	//https://blog.twitch.tv/de-de/2019/04/10/go-memory-ballast-how-i-learnt-to-stop-worrying-and-love-the-heap-26c2462549a2/
+	//ballast := make([]byte, 100<<20) //100MiB
+	//ballast[0] = 1
+	c := make(chan struct{}, 0)
+
+	println("WASM Go Initialized")
+	// register functions
+	registerCallbacks()
+	<-c
+}

model/Database.go

@@ -1,22 +1,15 @@
 package model
 
 import (
-	"archive/zip"
 	"database/sql"
 	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"reflect"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/pkg/errors"
 	"github.com/sergi/go-diff/diffmatchpatch"
-
-	// Register SQLite driver
-	_ "github.com/mattn/go-sqlite3"
 )
 
 const manifestFilename = "manifest.json"
@@ -174,36 +167,17 @@ func (db *Database) Equals(other *Database) bool {
 // ImportJWLBackup unzips a given JW Library Backup file and imports the
 // included SQLite DB to the Database struct
 func (db *Database) ImportJWLBackup(filename string) error {
+	pers := GetPersistence()
 	// Create tmp folder and place all files there
-	tmp, err := ioutil.TempDir("", "go-jwlm")
+	tmp, err := pers.CreateTempStorage("go-jwlm")
 	if err != nil {
-		return errors.Wrap(err, "Error while creating temporary directory")
+		return errors.Wrap(err, "Error while creating temp storage")
 	}
-	defer os.RemoveAll(tmp)
+	defer pers.CleanupPath(tmp)
 
-	r, err := zip.OpenReader(filename)
+	err = pers.ProcessJWLBackup(filename, tmp)
 	if err != nil {
-		return err
-	}
-	defer r.Close()
-
-	for _, file := range r.File {
-		fileReader, err := file.Open()
-		if err != nil {
-			return err
-		}
-		defer fileReader.Close()
-
-		path := filepath.Join(tmp, file.Name)
-		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
-		if err != nil {
-			return err
-		}
-		defer targetFile.Close()
-
-		if _, err := io.Copy(targetFile, fileReader); err != nil {
-			return errors.Wrap(err, "Error while copying files from backup to temporary folder")
-		}
+		return errors.Wrap(err, "Error while processing JW Library backup")
 	}
 
 	// Import manifest
@@ -226,7 +200,7 @@ func (db *Database) ImportJWLBackup(filename string) error {
 // importSQLite imports a given SQLite DB into the Database struct
 func (db *Database) importSQLite(filename string) error {
 	// Open SQLite file as immutable to avoid locks (and therefore speed up import)
-	sqlite, err := sql.Open("sqlite3", filename+"?immutable=1")
+	sqlite, err := GetPersistence().OpenSQLiteDB(filename + "?immutable=1")
 	if err != nil {
 		return errors.Wrap(err, "Error while opening SQLite database")
 	}
@@ -381,11 +355,12 @@ func getSliceCapacity(sqlite *sql.DB, modelType Model) (int, error) {
 // ExportJWLBackup creates a .jwlibrary backup file out of a Database{} struct
 func (db *Database) ExportJWLBackup(filename string) error {
 	// Create tmp folder and place all files there
-	tmp, err := ioutil.TempDir("", "go-jwlm")
+	pers := GetPersistence()
+	tmp, err := pers.CreateTempStorage("go-jwlm")
 	if err != nil {
-		return errors.Wrap(err, "Error while creating temporary directory")
+		return errors.Wrap(err, "Error while creating temp storage")
 	}
-	defer os.RemoveAll(tmp)
+	defer pers.CleanupPath(tmp)
 
 	// Create user_data.db
 	dbPath := filepath.Join(tmp, "user_data.db")
@@ -419,7 +394,7 @@ func (db *Database) saveToNewSQLite(filename string) error {
 		return errors.Wrap(err, "Error while creating new empty SQLite database")
 	}
 
-	sqlite, err := sql.Open("sqlite3", filename)
+	sqlite, err := GetPersistence().OpenSQLiteDB(filename)
 	if err != nil {
 		return errors.Wrap(err, "Error while opening SQLite database")
 	}
@@ -429,6 +404,7 @@ func (db *Database) saveToNewSQLite(filename string) error {
 	// and use it to insert its entries to the new SQLite DB
 	dbFields := reflect.ValueOf(db).Elem()
 	for j := 0; j < dbFields.NumField(); j++ {
+		fmt.Printf("Inserting %ss\n", dbFields.Type().Field(j).Name)
 		slice := dbFields.Field(j).Interface()
 		mdl, err := MakeModelSlice(slice)
 		if err != nil {
@@ -538,7 +514,7 @@ func createEmptySQLiteDB(filename string) error {
 		return errors.Wrap(err, "Error while fetching user_data.db")
 	}
 
-	if err := ioutil.WriteFile(filename, userData, 0644); err != nil {
+	if err := GetPersistence().WriteFile(filename, userData); err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Error while saving new SQLite database at %s", filename))
 	}
 

model/Persistence.go

@@ -0,0 +1,35 @@
+package model
+
+import (
+	"database/sql"
+	"runtime"
+	"sync"
+)
+
+var once sync.Once
+var persistence Persistence
+
+type Persistence interface {
+	CreateTempStorage(prefix string) (path string, err error)
+	StoreSQLiteDB(filename string, dbData []byte) (fullFileName string, err error)
+	OpenSQLiteDB(fullFileName string) (*sql.DB, error)
+	RetrieveSQLiteData(fullFileName string) ([]byte, error)
+	StoreJWLBackup(fullFileName string, archiveData []byte) error
+	ProcessJWLBackup(fullFileName string, exportPath string) error
+	GetFile(fullFileName string) (filename string, data []byte, err error)
+	WriteFile(fullFileName string, data []byte) error
+	CleanupPath(path string) error
+}
+
+func GetPersistence() Persistence {
+	once.Do(func() {
+		if runtime.GOOS == "js" {
+			persistence = getJsPersistence()
+		} else {
+			persistence = getFsPersistence()
+		}
+	})
+
+	return persistence
+
+}

model/fsPersistence.go

@@ -0,0 +1,116 @@
+// +build !js
+
+package model
+
+import (
+	"archive/zip"
+	"database/sql"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	// Register SQLite driver
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/pkg/errors"
+)
+
+type fsPersistence struct{}
+
+func getFsPersistence() Persistence {
+	pers := fsPersistence{}
+	return &pers
+}
+
+func getJsPersistence() Persistence {
+	panic("getJsPersistence call in non-js runtime")
+}
+
+func (pers *fsPersistence) CreateTempStorage(prefix string) (path string, err error) {
+	// Create tmp folder and place all files there
+	tmp, err := ioutil.TempDir("", prefix)
+	if err != nil {
+		return "", errors.Wrap(err, "Error while creating temporary directory")
+	}
+	return tmp, nil
+}
+
+func (pers *fsPersistence) StoreSQLiteDB(filename string, dbData []byte) (fullFileName string, err error) {
+	return "", errors.Errorf("Not needed to store the JWLBackup when using fsPersistence")
+}
+
+func (pers *fsPersistence) OpenSQLiteDB(fullFileName string) (*sql.DB, error) {
+	return sql.Open("sqlite3", fullFileName)
+}
+
+func (pers *fsPersistence) RetrieveSQLiteData(fullFileName string) ([]byte, error) {
+	_, data, err := pers.GetFile(fullFileName)
+	return data, err
+}
+
+func (pers *fsPersistence) StoreJWLBackup(fullFileName string, archiveData []byte) error {
+	return errors.Errorf("Not needed to store the JWLBackup when using fsPersistence")
+}
+
+func (pers *fsPersistence) ProcessJWLBackup(fullFileName string, exportPath string) error {
+
+	r, err := zip.OpenReader(fullFileName)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	for _, file := range r.File {
+		fileReader, err := file.Open()
+		if err != nil {
+			return errors.Wrap(err, "Error while opening zip file")
+		}
+		defer fileReader.Close()
+
+		path := filepath.Join(exportPath, file.Name)
+		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
+		if err != nil {
+			return errors.Wrap(err, "Error while uncompressing zip file")
+		}
+		defer targetFile.Close()
+
+		if _, err := io.Copy(targetFile, fileReader); err != nil {
+			return errors.Wrap(err, "Error while copying files from backup to folder")
+		}
+	}
+
+	return nil
+}
+
+func (pers *fsPersistence) GetFile(fullFileName string) (filename string, data []byte, err error) {
+	file, err := os.Open(fullFileName)
+	if err != nil {
+		return "", nil, errors.Wrap(err, fmt.Sprintf("Error opening file at %v", fullFileName))
+	}
+	defer file.Close()
+
+	blob, err := ioutil.ReadAll(file)
+	if err != nil {
+		return "", nil, errors.Wrap(err, fmt.Sprintf("Error reading file at %v", fullFileName))
+	}
+
+	return file.Name(), blob, nil
+
+}
+
+func (pers *fsPersistence) WriteFile(fullFileName string, data []byte) error {
+	if err := ioutil.WriteFile(fullFileName, data, 0644); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Error while saving file at %v", fullFileName))
+	}
+	return nil
+}
+
+func (pers *fsPersistence) CleanupPath(path string) error {
+	err := os.RemoveAll(path)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Error clearing path %v", path))
+	}
+
+	return nil
+}