diff --git a/.gitignore b/.gitignore index 4cd2df4..9eeed13 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ go-jwlm 2020-08-06.jwlibrary 2020-03-30.jwlibrary dist +*.exe +*.wasm \ No newline at end of file diff --git a/compileWasm.ps1 b/compileWasm.ps1 new file mode 100644 index 0000000..f361eeb Binary files /dev/null and b/compileWasm.ps1 differ diff --git a/go.mod b/go.mod index 36d3d58..520f604 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec github.com/codeclysm/extract/v3 v3.0.2 github.com/davecgh/go-spew v1.1.1 + github.com/fritzbauer/go-sqlite3-js v0.0.0-20210228162201-a9149e4032a8 github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-openapi/errors v0.19.9 // indirect github.com/go-openapi/strfmt v0.19.11 // indirect diff --git a/go.sum b/go.sum index f3d1638..f7bfc9e 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fritzbauer/go-sqlite3-js v0.0.0-20210227210033-af80406c0f5d h1:P52Q8OHtQfl3wECj7xzG8YIQ/4sieEk2e4fdlQNMpjA= +github.com/fritzbauer/go-sqlite3-js v0.0.0-20210227210033-af80406c0f5d/go.mod h1:4l4Wgw6/8enHFSGtRO5PAfMwcnPjL3JX8Wtp+N+geCU= +github.com/fritzbauer/go-sqlite3-js v0.0.0-20210228162201-a9149e4032a8 h1:GpYToYmFKCBFloxgGVrsw/b8CsrvI5x1lhhQ7zc/rKQ= +github.com/fritzbauer/go-sqlite3-js v0.0.0-20210228162201-a9149e4032a8/go.mod h1:4l4Wgw6/8enHFSGtRO5PAfMwcnPjL3JX8Wtp+N+geCU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -283,6 +287,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/main.go b/main.go index 1016ee7..f3a9fd6 100644 --- a/main.go +++ b/main.go @@ -1,3 +1,5 @@ +// +build !js + package main import "github.com/AndreasSko/go-jwlm/cmd" diff --git a/main_wasm.go b/main_wasm.go new file mode 100644 index 0000000..ad648c5 --- /dev/null +++ b/main_wasm.go @@ -0,0 +1,48 @@ +// +build js + +package main + +import ( + "fmt" + "syscall/js" + + "github.com/AndreasSko/go-jwlm/wasm" +) + +func mergeJs(this js.Value, inputs []js.Value) interface{} { + leftDbArr := inputs[0] + rightDbArr := inputs[1] + mergedDbName := inputs[2].String() + //mergedDbArr := make([]uint8, leftDbArr.Get("byteLength").Int()) + + leftBuf := make([]uint8, leftDbArr.Get("byteLength").Int()) + rightBuf := make([]uint8, rightDbArr.Get("byteLength").Int()) + + js.CopyBytesToGo(leftBuf, leftDbArr) + js.CopyBytesToGo(rightBuf, rightDbArr) + + mergedDb := wasm.Merge(leftBuf, rightBuf, mergedDbName) + + //js.CopyBytesToJS(mergedDbArr, mergedDb) + fmt.Printf("Merged. Returning %d bytes\n", len(mergedDb)) + mergedJsData := js.Global().Get("Uint8Array").New(len(mergedDb)) + js.CopyBytesToJS(mergedJsData, mergedDb) + return mergedJsData + +} + +func registerCallbacks() { + js.Global().Set("mergeJs", js.FuncOf(mergeJs)) +} + +func main() { + //https://blog.twitch.tv/de-de/2019/04/10/go-memory-ballast-how-i-learnt-to-stop-worrying-and-love-the-heap-26c2462549a2/ + //ballast := make([]byte, 100<<20) //100MiB + //ballast[0] = 1 + c := make(chan struct{}, 0) + + println("WASM Go Initialized") + // register functions + registerCallbacks() + <-c +} diff --git a/model/Database.go b/model/Database.go index 162059b..05f4dad 100644 --- a/model/Database.go +++ b/model/Database.go @@ -1,12 +1,8 @@ package model import ( - "archive/zip" "database/sql" "fmt" - "io" - "io/ioutil" - "os" "path/filepath" "reflect" "time" @@ -14,9 +10,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pkg/errors" "github.com/sergi/go-diff/diffmatchpatch" - - // Register SQLite driver - _ "github.com/mattn/go-sqlite3" ) const manifestFilename = "manifest.json" @@ -174,36 +167,17 @@ func (db *Database) Equals(other *Database) bool { // ImportJWLBackup unzips a given JW Library Backup file and imports the // included SQLite DB to the Database struct func (db *Database) ImportJWLBackup(filename string) error { + pers := GetPersistence() // Create tmp folder and place all files there - tmp, err := ioutil.TempDir("", "go-jwlm") + tmp, err := pers.CreateTempStorage("go-jwlm") if err != nil { - return errors.Wrap(err, "Error while creating temporary directory") + return errors.Wrap(err, "Error while creating temp storage") } - defer os.RemoveAll(tmp) + defer pers.CleanupPath(tmp) - r, err := zip.OpenReader(filename) + err = pers.ProcessJWLBackup(filename, tmp) if err != nil { - return err - } - defer r.Close() - - for _, file := range r.File { - fileReader, err := file.Open() - if err != nil { - return err - } - defer fileReader.Close() - - path := filepath.Join(tmp, file.Name) - targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) - if err != nil { - return err - } - defer targetFile.Close() - - if _, err := io.Copy(targetFile, fileReader); err != nil { - return errors.Wrap(err, "Error while copying files from backup to temporary folder") - } + return errors.Wrap(err, "Error while processing JW Library backup") } // Import manifest @@ -226,7 +200,7 @@ func (db *Database) ImportJWLBackup(filename string) error { // importSQLite imports a given SQLite DB into the Database struct func (db *Database) importSQLite(filename string) error { // Open SQLite file as immutable to avoid locks (and therefore speed up import) - sqlite, err := sql.Open("sqlite3", filename+"?immutable=1") + sqlite, err := GetPersistence().OpenSQLiteDB(filename + "?immutable=1") if err != nil { return errors.Wrap(err, "Error while opening SQLite database") } @@ -381,11 +355,12 @@ func getSliceCapacity(sqlite *sql.DB, modelType Model) (int, error) { // ExportJWLBackup creates a .jwlibrary backup file out of a Database{} struct func (db *Database) ExportJWLBackup(filename string) error { // Create tmp folder and place all files there - tmp, err := ioutil.TempDir("", "go-jwlm") + pers := GetPersistence() + tmp, err := pers.CreateTempStorage("go-jwlm") if err != nil { - return errors.Wrap(err, "Error while creating temporary directory") + return errors.Wrap(err, "Error while creating temp storage") } - defer os.RemoveAll(tmp) + defer pers.CleanupPath(tmp) // Create user_data.db dbPath := filepath.Join(tmp, "user_data.db") @@ -419,7 +394,7 @@ func (db *Database) saveToNewSQLite(filename string) error { return errors.Wrap(err, "Error while creating new empty SQLite database") } - sqlite, err := sql.Open("sqlite3", filename) + sqlite, err := GetPersistence().OpenSQLiteDB(filename) if err != nil { return errors.Wrap(err, "Error while opening SQLite database") } @@ -429,6 +404,7 @@ func (db *Database) saveToNewSQLite(filename string) error { // and use it to insert its entries to the new SQLite DB dbFields := reflect.ValueOf(db).Elem() for j := 0; j < dbFields.NumField(); j++ { + fmt.Printf("Inserting %ss\n", dbFields.Type().Field(j).Name) slice := dbFields.Field(j).Interface() mdl, err := MakeModelSlice(slice) if err != nil { @@ -538,7 +514,7 @@ func createEmptySQLiteDB(filename string) error { return errors.Wrap(err, "Error while fetching user_data.db") } - if err := ioutil.WriteFile(filename, userData, 0644); err != nil { + if err := GetPersistence().WriteFile(filename, userData); err != nil { return errors.Wrap(err, fmt.Sprintf("Error while saving new SQLite database at %s", filename)) } diff --git a/model/Persistence.go b/model/Persistence.go new file mode 100644 index 0000000..ca673ed --- /dev/null +++ b/model/Persistence.go @@ -0,0 +1,35 @@ +package model + +import ( + "database/sql" + "runtime" + "sync" +) + +var once sync.Once +var persistence Persistence + +type Persistence interface { + CreateTempStorage(prefix string) (path string, err error) + StoreSQLiteDB(filename string, dbData []byte) (fullFileName string, err error) + OpenSQLiteDB(fullFileName string) (*sql.DB, error) + RetrieveSQLiteData(fullFileName string) ([]byte, error) + StoreJWLBackup(fullFileName string, archiveData []byte) error + ProcessJWLBackup(fullFileName string, exportPath string) error + GetFile(fullFileName string) (filename string, data []byte, err error) + WriteFile(fullFileName string, data []byte) error + CleanupPath(path string) error +} + +func GetPersistence() Persistence { + once.Do(func() { + if runtime.GOOS == "js" { + persistence = getJsPersistence() + } else { + persistence = getFsPersistence() + } + }) + + return persistence + +} diff --git a/model/fsPersistence.go b/model/fsPersistence.go new file mode 100644 index 0000000..297368d --- /dev/null +++ b/model/fsPersistence.go @@ -0,0 +1,116 @@ +// +build !js + +package model + +import ( + "archive/zip" + "database/sql" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + // Register SQLite driver + _ "github.com/mattn/go-sqlite3" + "github.com/pkg/errors" +) + +type fsPersistence struct{} + +func getFsPersistence() Persistence { + pers := fsPersistence{} + return &pers +} + +func getJsPersistence() Persistence { + panic("getJsPersistence call in non-js runtime") +} + +func (pers *fsPersistence) CreateTempStorage(prefix string) (path string, err error) { + // Create tmp folder and place all files there + tmp, err := ioutil.TempDir("", prefix) + if err != nil { + return "", errors.Wrap(err, "Error while creating temporary directory") + } + return tmp, nil +} + +func (pers *fsPersistence) StoreSQLiteDB(filename string, dbData []byte) (fullFileName string, err error) { + return "", errors.Errorf("Not needed to store the JWLBackup when using fsPersistence") +} + +func (pers *fsPersistence) OpenSQLiteDB(fullFileName string) (*sql.DB, error) { + return sql.Open("sqlite3", fullFileName) +} + +func (pers *fsPersistence) RetrieveSQLiteData(fullFileName string) ([]byte, error) { + _, data, err := pers.GetFile(fullFileName) + return data, err +} + +func (pers *fsPersistence) StoreJWLBackup(fullFileName string, archiveData []byte) error { + return errors.Errorf("Not needed to store the JWLBackup when using fsPersistence") +} + +func (pers *fsPersistence) ProcessJWLBackup(fullFileName string, exportPath string) error { + + r, err := zip.OpenReader(fullFileName) + if err != nil { + return err + } + defer r.Close() + + for _, file := range r.File { + fileReader, err := file.Open() + if err != nil { + return errors.Wrap(err, "Error while opening zip file") + } + defer fileReader.Close() + + path := filepath.Join(exportPath, file.Name) + targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) + if err != nil { + return errors.Wrap(err, "Error while uncompressing zip file") + } + defer targetFile.Close() + + if _, err := io.Copy(targetFile, fileReader); err != nil { + return errors.Wrap(err, "Error while copying files from backup to folder") + } + } + + return nil +} + +func (pers *fsPersistence) GetFile(fullFileName string) (filename string, data []byte, err error) { + file, err := os.Open(fullFileName) + if err != nil { + return "", nil, errors.Wrap(err, fmt.Sprintf("Error opening file at %v", fullFileName)) + } + defer file.Close() + + blob, err := ioutil.ReadAll(file) + if err != nil { + return "", nil, errors.Wrap(err, fmt.Sprintf("Error reading file at %v", fullFileName)) + } + + return file.Name(), blob, nil + +} + +func (pers *fsPersistence) WriteFile(fullFileName string, data []byte) error { + if err := ioutil.WriteFile(fullFileName, data, 0644); err != nil { + return errors.Wrap(err, fmt.Sprintf("Error while saving file at %v", fullFileName)) + } + return nil +} + +func (pers *fsPersistence) CleanupPath(path string) error { + err := os.RemoveAll(path) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Error clearing path %v", path)) + } + + return nil +} diff --git a/model/jsPersistence.go b/model/jsPersistence.go new file mode 100644 index 0000000..692c511 --- /dev/null +++ b/model/jsPersistence.go @@ -0,0 +1,203 @@ +// +build js + +package model + +import ( + // Register SQLite driver + "archive/zip" + "bufio" + "bytes" + "database/sql" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "syscall/js" + + _ "github.com/fritzbauer/go-sqlite3-js" + "github.com/pkg/errors" +) + +const jsSqliteVarPrefix = "_go-jwlm_db" + +type jsPersistence struct { + storage map[string]*PersistedFolder +} + +type PersistedFolder struct { + Files map[string]*PersistedFile +} + +type PersistedFile struct { + Name string + Data []byte +} + +func getJsPersistence() Persistence { + pers := jsPersistence{storage: make(map[string]*PersistedFolder)} + return &pers +} + +func getFsPersistence() Persistence { + panic("getFsPersistence call in js runtime") +} + +func (pers *jsPersistence) CreateTempStorage(prefix string) (path string, err error) { + //find the first unused foldername + var i int + for i = 0; pers.storage[fmt.Sprintf("%s_%d", prefix, i)] != nil; i++ { + } + path = fmt.Sprintf("%s_%d", prefix, i) + pers.storage[path] = &PersistedFolder{Files: make(map[string]*PersistedFile)} + return path, nil +} + +func (pers *jsPersistence) StoreSQLiteDB(filename string, dbData []byte) (fullFileName string, err error) { + //TODO think of proper errorhandling + jsName := fmt.Sprintf("%s_%d_%s", jsSqliteVarPrefix, 0, filename) + + //prevent using an already used JS variable + for i := 1; js.Global().Get(jsName).Truthy(); i++ { + jsName = fmt.Sprintf("%s_%d_%s", jsSqliteVarPrefix, i, filename) + } + + arr := js.Global().Get("Uint8Array").New(len(dbData)) + js.CopyBytesToJS(arr, dbData) + js.Global().Set(jsName, arr) + return jsName, nil +} + +func (pers *jsPersistence) OpenSQLiteDB(fullFileName string) (*sql.DB, error) { + if strings.Contains(fullFileName, "?") { //remove ?immutable=1 + fullFileName = strings.Split(fullFileName, "?")[0] + } + _, data, err := pers.GetFile(fullFileName) + jsStorageVariableName, err := pers.StoreSQLiteDB(fullFileName, data) + if err != nil { + return nil, errors.Wrap(err, "Could not store SQLite db") + } + + //add a "file" to store the mapping of filepath to js var name + err = pers.WriteFile(fullFileName, []byte(jsStorageVariableName)) + if err != nil { + return nil, errors.Wrap(err, "Error storing jsStorageVariableName") + } + + return sql.Open("sqlite3_js", fmt.Sprintf("jsvar:%s", jsStorageVariableName)) +} + +func (pers *jsPersistence) RetrieveSQLiteData(jsVarName string) ([]byte, error) { + //TODO think of proper errorhandling + dbMap := js.Global().Get("_go_sqlite_dbs") + jsData := dbMap.Call("get", jsVarName).Call("export") + data := make([]byte, jsData.Get("byteLength").Int()) + js.CopyBytesToGo(data, jsData) + return data, nil +} + +func (pers *jsPersistence) StoreJWLBackup(fullFileName string, archiveData []byte) error { + path, fileName := evaluateFullFileName(fullFileName) + + folder, ok := pers.storage[path] + if !ok { + pers.storage[path] = &PersistedFolder{Files: make(map[string]*PersistedFile)} + folder = pers.storage[path] + } + + folder.Files[fileName] = &PersistedFile{Name: fileName, Data: archiveData} + //pers.printStorage() //debug + return nil +} + +func (pers *jsPersistence) ProcessJWLBackup(fullFileName string, exportPath string) error { + + _, data, err := pers.GetFile(fullFileName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Could not find JW Library backup at %v", fullFileName)) + } + reader := bytes.NewReader(data) + + r, err := zip.NewReader(reader, int64(len(data))) + if err != nil { + return errors.Wrap(err, "Could not read zip") + } + + for _, file := range r.File { + fileReader, err := file.Open() + if err != nil { + return errors.Wrap(err, "Error while opening zip file") + } + defer fileReader.Close() + + var buf bytes.Buffer + _, err = io.Copy(bufio.NewWriter(&buf), fileReader) + if err != nil { + return errors.Wrap(err, "Error while storing files from backup ") + } + + path := filepath.Join(exportPath, file.Name) + pers.WriteFile(path, buf.Bytes()) + } + //pers.printStorage() //debug + return nil +} + +func (pers *jsPersistence) GetFile(fullFileName string) (filename string, data []byte, err error) { + path, fileName := evaluateFullFileName(fullFileName) + file := pers.storage[path].Files[fileName] + if file == nil { + return "", nil, errors.Errorf("Could not find file '%v' at %v", fileName, path) + } + + if strings.HasPrefix(string(file.Data), jsSqliteVarPrefix) { + mergedData, err := pers.RetrieveSQLiteData(string(file.Data)) + if err != nil { + return "", nil, errors.Wrap(err, "Error while getting SQLite DB") + } + return fileName, mergedData, nil + } + + //fmt.Printf("Returning %s; Length: %d\n", file.Name, len(file.Data)) //debug + return file.Name, file.Data, nil +} + +func (pers *jsPersistence) WriteFile(fullFileName string, data []byte) error { + path, fileName := evaluateFullFileName(fullFileName) + + folder := pers.getFolder(path) + + folder.Files[fileName] = &PersistedFile{Name: fileName, Data: data} //this is silently overwriting any existing file + return nil +} + +func (pers *jsPersistence) CleanupPath(path string) error { + delete(pers.storage, path) + return nil +} + +func (pers *jsPersistence) getFolder(path string) *PersistedFolder { + folder, ok := pers.storage[path] + if !ok { + pers.storage[path] = &PersistedFolder{Files: make(map[string]*PersistedFile)} + folder = pers.storage[path] + } + return folder +} + +func evaluateFullFileName(fullFileName string) (path string, fileName string) { + fullfileNameParts := strings.Split(fullFileName, string(os.PathSeparator)) + path = strings.Join(fullfileNameParts[:len(fullfileNameParts)-1], string(os.PathSeparator)) + fileName = fullfileNameParts[len(fullfileNameParts)-1] + //fmt.Printf("Splitted '%s' into '%s' and '%s'\n", fullFileName, path, fileName) //debug + return path, fileName +} + +//Debugging print +func (pers *jsPersistence) printStorage() { + for folderName, folder := range pers.storage { + for filename, file := range folder.Files { + fmt.Printf("PrintStorage: %s/%s File.Name: %s; Length: %d\n", folderName, filename, file.Name, len(file.Data)) + } + } +} diff --git a/model/manifest.go b/model/manifest.go index d8c5440..ab3ee2e 100644 --- a/model/manifest.go +++ b/model/manifest.go @@ -4,10 +4,6 @@ import ( "crypto/sha256" "encoding/json" "fmt" - "io" - "io/ioutil" - "log" - "os" "path/filepath" "time" @@ -31,15 +27,12 @@ type userDataBackup struct { // importManifest imports a manifest.json at path func (mfst *manifest) importManifest(path string) error { - file, err := os.Open(path) + _, data, err := GetPersistence().GetFile(path) if err != nil { - return err + return errors.Wrap(err, "Error loading manifest file") } - defer file.Close() - blob, _ := ioutil.ReadAll(file) - - err = json.Unmarshal([]byte(blob), &mfst) + err = json.Unmarshal(data, &mfst) if err != nil { return errors.Wrap(err, "Could not unmarshall backup manifest file") } @@ -69,16 +62,11 @@ func (mfst *manifest) validateManifest() error { // later be exported func generateManifest(backupName string, dbFile string) (*manifest, error) { // Get SHA256 of SQLite file - f, err := os.Open(dbFile) + _, data, err := GetPersistence().GetFile(dbFile) if err != nil { return nil, errors.Wrapf(err, "Error while opening SQLite file %s to calculate hash", dbFile) } - defer f.Close() - hasher := sha256.New() - if _, err := io.Copy(hasher, f); err != nil { - log.Fatal(err) - } - hash := fmt.Sprintf("%x", hasher.Sum(nil)) + hash := fmt.Sprintf("%x", sha256.Sum256(data)) mfst := &manifest{ CreationDate: time.Now().Format("2006-01-02"), @@ -104,7 +92,8 @@ func (mfst *manifest) exportManifest(path string) error { return errors.Wrap(err, "Error while marshalling manifest") } - if err := ioutil.WriteFile(path, bytes, 0644); err != nil { + err = GetPersistence().WriteFile(path, bytes) + if err != nil { return errors.Wrap(err, fmt.Sprintf("Error while saving manifest file at %v", path)) } diff --git a/model/zip.go b/model/zip.go index 92907a4..dfc41f3 100644 --- a/model/zip.go +++ b/model/zip.go @@ -2,58 +2,52 @@ package model import ( "archive/zip" + "bytes" "io" - "os" - "path/filepath" + "time" + + "github.com/pkg/errors" ) // https://golangcode.com/create-zip-files-in-go/ func zipFiles(filename string, files []string) error { - - newZipFile, err := os.Create(filename) - if err != nil { - return err - } - defer newZipFile.Close() - - zipWriter := zip.NewWriter(newZipFile) - defer zipWriter.Close() + var zipBuffer bytes.Buffer + zipWriter := zip.NewWriter(&zipBuffer) // Add files to zip for _, file := range files { - if err = addFileToZip(zipWriter, file); err != nil { + if err := addFileToZip(zipWriter, file); err != nil { + zipWriter.Close() return err } } - return nil -} - -func addFileToZip(zipWriter *zip.Writer, filename string) error { - fileToZip, err := os.Open(filename) + zipWriter.Close() + err := GetPersistence().WriteFile(filename, zipBuffer.Bytes()) if err != nil { - return err - } - defer fileToZip.Close() - // Get the file information - info, err := fileToZip.Stat() - if err != nil { - return err + return errors.Wrap(err, "Error storing zip") } - header, err := zip.FileInfoHeader(info) + return nil +} + +func addFileToZip(zipWriter *zip.Writer, filename string) error { + + name, data, err := GetPersistence().GetFile(filename) if err != nil { return err } - header.Name = filepath.Base(filename) + header := zip.FileHeader{} + header.Name = name header.Method = zip.Deflate + header.Modified = time.Now() - writer, err := zipWriter.CreateHeader(header) + writer, err := zipWriter.CreateHeader(&header) if err != nil { return err } - _, err = io.Copy(writer, fileToZip) + _, err = io.Copy(writer, bytes.NewReader(data)) return err } diff --git a/wasm/merge.go b/wasm/merge.go new file mode 100644 index 0000000..0693364 --- /dev/null +++ b/wasm/merge.go @@ -0,0 +1,215 @@ +//+build js + +package wasm + +import ( + "fmt" + "os" + + "github.com/AndreasSko/go-jwlm/merger" + "github.com/AndreasSko/go-jwlm/model" + "github.com/pkg/errors" + + log "github.com/sirupsen/logrus" +) + +// BookmarkResolver represents a resolver that should be used for conflicting Bookmarks +var BookmarkResolver string + +// MarkingResolver represents a resolver that should be used for conflicting UserMarkBlockRanges +var MarkingResolver string + +// NoteResolver represents a resolver that should be used for conflicting Notes +var NoteResolver string + +func Merge(leftFile []byte, rightFile []byte, mergedFilename string) []byte { + BookmarkResolver = "chooseLeft" //chooseNewest chooseLeft chooseRight + MarkingResolver = "chooseLeft" + NoteResolver = "chooseLeft" + + pers := model.GetPersistence() + tmpPath, err := pers.CreateTempStorage("jwlBackups") + if err != nil { + log.Fatal(errors.Wrap(err, "Error creating temp path")) + } + defer pers.CleanupPath(tmpPath) + //fmt.Println("Importing left backup", leftFilename) + + leftFullFileName := tmpPath + string(os.PathSeparator) + "left.jwlibrary" + err = pers.StoreJWLBackup(leftFullFileName, leftFile) + if err != nil { + log.Fatal(errors.Wrap(err, "Error storing left backup")) + } + //leftFilename := left.StoreBackupData(leftFile) + left := model.Database{} + err = left.ImportJWLBackup(leftFullFileName) + if err != nil { + log.Fatal(err) + } + + //fmt.Println("Importing right backup", rightFilename) + + rightFullFileName := tmpPath + string(os.PathSeparator) + "right.jwlibrary" + err = pers.StoreJWLBackup(rightFullFileName, rightFile) + //rightFilename := right.StoreBackupData(rightFile) + right := model.Database{} + err = right.ImportJWLBackup(rightFullFileName) + if err != nil { + log.Fatal(err) + } + + merged := model.Database{} + + fmt.Println("🧭 Merging Locations") + mergedLocations, locationIDChanges, err := merger.MergeLocations(left.Location, right.Location) + merged.Location = mergedLocations + merger.UpdateLRIDs(left.Bookmark, right.Bookmark, "LocationID", locationIDChanges) + merger.UpdateLRIDs(left.Bookmark, right.Bookmark, "PublicationLocationID", locationIDChanges) + merger.UpdateLRIDs(left.Note, right.Note, "LocationID", locationIDChanges) + merger.UpdateLRIDs(left.TagMap, right.TagMap, "LocationID", locationIDChanges) + merger.UpdateLRIDs(left.UserMark, right.UserMark, "LocationID", locationIDChanges) + fmt.Println("Done.") + + fmt.Println("📑 Merging Bookmarks") + bookmarksConflictSolution := map[string]merger.MergeSolution{} + for { + mergedBookmarks, _, err := merger.MergeBookmarks(left.Bookmark, right.Bookmark, bookmarksConflictSolution) + if err == nil { + merged.Bookmark = mergedBookmarks + break + } + switch err := err.(type) { + case merger.MergeConflictError: + if BookmarkResolver != "" { + var resErr error + newSolutions, resErr := merger.AutoResolveConflicts(err.Conflicts, BookmarkResolver) + if resErr != nil { + log.Fatal(resErr) + } + addToSolutions(bookmarksConflictSolution, newSolutions) + } /*else { + newSolutions := handleMergeConflict(err.Conflicts, &merged, stdio) + addToSolutions(bookmarksConflictSolution, newSolutions) + }*/ + default: + log.Fatal(err) + } + } + fmt.Println("Done.") + + fmt.Println("🏷 Merging Tags") + var tagsConflictSolution map[string]merger.MergeSolution + for { + mergedTags, tagIDChanges, err := merger.MergeTags(left.Tag, right.Tag, tagsConflictSolution) + if err == nil { + merged.Tag = mergedTags + merger.UpdateLRIDs(left.TagMap, right.TagMap, "TagID", tagIDChanges) + break + } + switch err := err.(type) { + case merger.MergeConflictError: + //tagsConflictSolution = handleMergeConflict(err.Conflicts, nil, stdio) // TODO + default: + log.Fatal(err) + } + } + fmt.Println("Done.") + + fmt.Println("🖍 Merging Markings") + UMBRConflictSolution := map[string]merger.MergeSolution{} + for { + mergedUserMarks, mergedBlockRanges, userMarkIDChanges, err := merger.MergeUserMarkAndBlockRange(left.UserMark, left.BlockRange, right.UserMark, right.BlockRange, UMBRConflictSolution) + if err == nil { + merged.UserMark = mergedUserMarks + merged.BlockRange = mergedBlockRanges + merger.UpdateLRIDs(left.Note, right.Note, "UserMarkID", userMarkIDChanges) + break + } + switch err := err.(type) { + case merger.MergeConflictError: + if MarkingResolver != "" { + var resErr error + newSolutions, resErr := merger.AutoResolveConflicts(err.Conflicts, MarkingResolver) + if resErr != nil { + log.Fatal(resErr) + } + addToSolutions(UMBRConflictSolution, newSolutions) + } /*else { + newSolutions := handleMergeConflict(err.Conflicts, &merged, stdio) + addToSolutions(UMBRConflictSolution, newSolutions) + }*/ + default: + log.Fatal(err) + } + } + fmt.Println("Done.") + + fmt.Println("📝 Merging Notes") + notesConflictSolution := map[string]merger.MergeSolution{} + for { + mergedNotes, notesIDChanges, err := merger.MergeNotes(left.Note, right.Note, notesConflictSolution) + if err == nil { + merged.Note = mergedNotes + merger.UpdateLRIDs(left.TagMap, right.TagMap, "NoteID", notesIDChanges) + break + } + switch err := err.(type) { + case merger.MergeConflictError: + if NoteResolver != "" { + var resErr error + newSolutions, resErr := merger.AutoResolveConflicts(err.Conflicts, NoteResolver) + if resErr != nil { + log.Fatal(resErr) + } + addToSolutions(notesConflictSolution, newSolutions) + } /* else { + newSolutions := handleMergeConflict(err.Conflicts, &merged, stdio) + addToSolutions(notesConflictSolution, newSolutions) + }*/ + default: + log.Fatal(err) + } + } + fmt.Println("Done.") + + fmt.Println("🏷 Merging TagMaps") + var tagMapsConflictSolution map[string]merger.MergeSolution + for { + mergedTagMaps, _, err := merger.MergeTagMaps(left.TagMap, right.TagMap, tagMapsConflictSolution) + if err == nil { + merged.TagMap = mergedTagMaps + break + } + switch err := err.(type) { + case merger.MergeConflictError: + //tagMapsConflictSolution = handleMergeConflict(err.Conflicts, nil, stdio) + default: + log.Fatal(err) + } + } + fmt.Println("Done.") + + fmt.Println("🎉 Finished merging!") + + fmt.Println("Exporting merged database") + mergedPath := tmpPath + string(os.PathSeparator) + mergedFilename + err = merged.ExportJWLBackup(mergedPath) + if err != nil { + log.Fatal(err) + } + + _, mergedData, err := pers.GetFile(mergedPath) + if err != nil { + log.Fatal(errors.Wrap(err, "Error retrieving merged file.")) + } + + return mergedData + +} + +// addToSolutions adds new mergeSolutions to the existing map of mergeSolutions +func addToSolutions(solutions map[string]merger.MergeSolution, new map[string]merger.MergeSolution) { + for key, value := range new { + solutions[key] = value + } +}