feat: CGO FREE SQLITE添加

PaienNate · Jan 17, 2025 · 7f1218c · 7f1218c
1 parent 11c3456
commit 7f1218c
Show file tree

Hide file tree

Showing 5 changed files with 184 additions and 52 deletions.
diff --git a/dice/model/database/sqlite.go b/dice/model/database/sqlite.go
@@ -4,35 +4,162 @@
 package database
 
 import (
-	"github.com/glebarez/sqlite"
+	"database/sql"
+	"fmt"
+	"runtime"
+
+	_ "github.com/ncruces/go-sqlite3/embed"
+	sqlite "github.com/ncruces/go-sqlite3/gormlite"
+
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 
 	"sealdice-core/dice/model/database/cache"
 )
 
+// 警告：不要在一个事务（写事务）里使用读的DB！否则读的DB会发现有人在写而锁住，从而死锁。
+
 func SQLiteDBInit(path string, useWAL bool) (*gorm.DB, error) {
-	db, err := gorm.Open(sqlite.Open(path), &gorm.Config{
+	// 使用即时事务
+	path = fmt.Sprintf("file:%v?_txlock=immediate&_busy_timeout=15000", path)
+	open, err := gorm.Open(sqlite.Open(path), &gorm.Config{
 		// 注意，这里虽然是Info,但实际上打印就变成了Debug.
 		Logger: logger.Default.LogMode(logger.Info),
 	})
-	// https://github.com/glebarez/sqlite/issues/52 尚未遇见问题，可以先考虑不使用
-	// sqlDB, _ := db.DB()
-	// sqlDB.SetMaxOpenConns(1)
 	if err != nil {
 		return nil, err
 	}
 	// Enable Cache Mode
-	db, err = cache.GetOtterCacheDB(db)
+	open, err = cache.GetOtterCacheDB(open)
 	if err != nil {
 		return nil, err
 	}
-	// enable WAL mode
 	if useWAL {
-		err = db.Exec("PRAGMA journal_mode=WAL").Error
+		err = open.Exec("PRAGMA journal_mode=WAL").Error
 		if err != nil {
-			return nil, err
+			panic(err)
 		}
 	}
-	return db, err
+	return open, err
+}
+
+func createReadDB(path string, gormConf *gorm.Config) (*gorm.DB, error) {
+	// _txlock=immediate 解决BEGIN IMMEDIATELY
+	path = fmt.Sprintf("file:%v?_txlock=immediate", path)
+	// ---- 创建读连接 -----
+	readDB, err := gorm.Open(sqlite.Open(path), gormConf)
+	if err != nil {
+		return nil, err
+	}
+	readPool, err := readDB.DB()
+	if err != nil {
+		return nil, err
+	}
+	err = SetDefaultPragmas(readPool)
+	if err != nil {
+		return nil, err
+	}
+	readPool.SetMaxOpenConns(max(4, runtime.NumCPU()))
+	return readDB, nil
+}
+
+func createWriteDB(path string, gormConf *gorm.Config) (*gorm.DB, error) {
+	// 注意基于wasm的版本必须添加file:
+	path = fmt.Sprintf("file:%v?_txlock=immediate", path)
+	// ---- 创建写连接 -----
+	writeDB, err := gorm.Open(sqlite.Open(path), gormConf)
+	if err != nil {
+		return nil, err
+	}
+	writePool, err := writeDB.DB()
+	if err != nil {
+		return nil, err
+	}
+	err = SetDefaultPragmas(writePool)
+	if err != nil {
+		return nil, err
+	}
+	writePool.SetMaxOpenConns(1) // only use one active connection for writing
+	return writeDB, nil
+}
+
+func SQLiteDBRWInit(path string) (*gorm.DB, *gorm.DB, error) {
+	// 由于现在我们只有一个写入连接，所以不需要使用事务
+	gormConf := &gorm.Config{
+		Logger:                 logger.Default.LogMode(logger.Info),
+		SkipDefaultTransaction: true,
+	}
+	readDB, err := createReadDB(path, gormConf)
+	if err != nil {
+		return nil, nil, err
+	}
+	writeDB, err := createWriteDB(path, gormConf)
+	if err != nil {
+		return nil, nil, err
+	}
+	// ----- 启用共享缓存插件 -----
+	plugin, err := cache.GetOtterCacheDBPluginInstance()
+	if err != nil {
+		return nil, nil, err
+	}
+	err = readDB.Use(plugin)
+	if err != nil {
+		return nil, nil, err
+	}
+	err = writeDB.Use(plugin)
+	if err != nil {
+		return nil, nil, err
+	}
+	return readDB, writeDB, nil
+}
+
+// SetDefaultPragmas defines some sqlite pragmas for good performance and litestream compatibility
+// https://highperformancesqlite.com/articles/sqlite-recommended-pragmas
+// https://litestream.io/tips/
+// copied from https://github.com/bihe/monorepo
+// add PRAGMA optimize=0x10002; from https://github.com/Palats/mastopoof
+func SetDefaultPragmas(db *sql.DB) error {
+	var (
+		stmt string
+		val  string
+	)
+	// 外键的暂时弃用，反正咱也不用外键536870912
+	// "foreign_keys": "1",     // 1(bool) --> https://www.sqlite.org/pragma.html#pragma_foreign_keys
+	defaultPragmas := map[string]string{
+		"journal_mode": "wal",   // https://www.sqlite.org/pragma.html#pragma_journal_mode
+		"busy_timeout": "15000", // https://www.sqlite.org/pragma.html#pragma_busy_timeout
+		// 在 WAL 模式下使用 synchronous=NORMAL 提交的事务可能会在断电或系统崩溃后回滚。
+		// 无论同步设置或日志模式如何，事务在应用程序崩溃时都是持久的。
+		// 对于在 WAL 模式下运行的大多数应用程序来说，synchronous=NORMAL 设置是一个不错的选择。
+		"synchronous": "1",         // NORMAL --> https://www.sqlite.org/pragma.html#pragma_synchronous
+		"cache_size":  "536870912", // 536870912 = 512MB --> https://www.sqlite.org/pragma.html#pragma_cache_size
+	}
+
+	// set the pragmas
+	for k := range defaultPragmas {
+		stmt = fmt.Sprintf("pragma %s = %s", k, defaultPragmas[k])
+		if _, err := db.Exec(stmt); err != nil {
+			return err
+		}
+	}
+
+	// validate the pragmas
+	for k := range defaultPragmas {
+		row := db.QueryRow(fmt.Sprintf("pragma %s", k))
+		err := row.Scan(&val)
+		if err != nil {
+			return err
+		}
+		if val != defaultPragmas[k] {
+			return fmt.Errorf("could not set pragma %s to %s", k, defaultPragmas[k])
+		}
+	}
+	// 这个不能在上面，因为他没有任何返回值
+	// Setup some regular optimization according to sqlite doc:
+	//  https://www.sqlite.org/lang_analyze.html
+	if _, err := db.Exec("PRAGMA optimize=0x10002;"); err != nil {
+		return fmt.Errorf("unable set optimize pragma: %w", err)
+	}
+
+	return nil
 }
diff --git a/dice/model/database/sqlite_cgo.go b/dice/model/database/sqlite_cgo.go
@@ -19,7 +19,7 @@ import (
 
 func SQLiteDBInit(path string, useWAL bool) (*gorm.DB, error) {
 	// 使用即时事务
-	path = fmt.Sprintf("%v?_txlock=immediate&_busy_timeout=15000", path)
+	path = fmt.Sprintf("file:%v?_txlock=immediate&_busy_timeout=15000", path)
 	open, err := gorm.Open(sqlite.Open(path), &gorm.Config{
 		// 注意，这里虽然是Info,但实际上打印就变成了Debug.
 		Logger: logger.Default.LogMode(logger.Info),
@@ -43,7 +43,7 @@ func SQLiteDBInit(path string, useWAL bool) (*gorm.DB, error) {
 
 func createReadDB(path string, gormConf *gorm.Config) (*gorm.DB, error) {
 	// _txlock=immediate 解决BEGIN IMMEDIATELY
-	path = fmt.Sprintf("%v?_txlock=immediate", path)
+	path = fmt.Sprintf("file:%v?_txlock=immediate", path)
 	// ---- 创建读连接 -----
 	readDB, err := gorm.Open(sqlite.Open(path), gormConf)
 	if err != nil {
@@ -63,6 +63,7 @@ func createReadDB(path string, gormConf *gorm.Config) (*gorm.DB, error) {
 
 func createWriteDB(path string, gormConf *gorm.Config) (*gorm.DB, error) {
 	// ---- 创建写连接 -----
+	path = fmt.Sprintf("file:%v?_txlock=immediate", path)
 	writeDB, err := gorm.Open(sqlite.Open(path), gormConf)
 	if err != nil {
 		return nil, err

diff --git a/go.mod b/go.mod
@@ -22,8 +22,6 @@ require (
 	github.com/fy0/systray v1.2.2
 	github.com/fyrchik/go-shlex v0.0.0-20210215145004-cd7f49bfd959
 	github.com/gen2brain/beeep v0.0.0-20230907135156-1a38885a97fc
-	github.com/glebarez/go-sqlite v1.22.0
-	github.com/glebarez/sqlite v1.11.0
 	github.com/go-creed/sat v1.0.3
 	github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1
 	github.com/golang-module/carbon v1.7.3
@@ -67,9 +65,9 @@ require (
 	github.com/yuin/goldmark v1.7.4
 	go.etcd.io/bbolt v1.3.11
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.29.0
+	golang.org/x/crypto v0.32.0
 	golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8
-	golang.org/x/sys v0.27.0
+	golang.org/x/sys v0.29.0
 	golang.org/x/text v0.21.0
 	golang.org/x/time v0.5.0
 	gopkg.in/elazarl/goproxy.v1 v1.0.0-20180725130230-947c36da3153
@@ -81,11 +79,11 @@ require (
 
 require (
 	github.com/blevesearch/bleve_index_api v1.1.13
+	github.com/glebarez/sqlite v1.11.0
 	github.com/go-gorm/caches/v4 v4.0.5
 	github.com/gofrs/flock v0.12.1
-	github.com/gonutz/w32/v2 v2.11.1
 	github.com/joho/godotenv v1.5.1
-	github.com/spaolacci/murmur3 v1.1.0
+	github.com/maypok86/otter v1.2.4
 	github.com/tidwall/gjson v1.17.0
 	github.com/tidwall/sjson v1.2.5
 	gorm.io/driver/mysql v1.5.7
@@ -128,6 +126,7 @@ require (
 	github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7 // indirect
 	github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55 // indirect
 	github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f // indirect
+	github.com/glebarez/go-sqlite v1.21.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-resty/resty/v2 v2.11.0 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
@@ -157,12 +156,14 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/maypok86/otter v1.2.4 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
+	github.com/ncruces/go-sqlite3 v0.22.0 // indirect
+	github.com/ncruces/go-sqlite3/gormlite v0.22.0 // indirect
+	github.com/ncruces/julianday v1.0.0 // indirect
 	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
 	github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
 	github.com/pdfcpu/pdfcpu v0.8.1 // indirect
@@ -177,6 +178,7 @@ require (
 	github.com/sunshineplan/tiff v0.0.0-20220128141034-29b9d69bd906 // indirect
 	github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect
 	github.com/tdewolff/parse/v2 v2.7.15 // indirect
+	github.com/tetratelabs/wazero v1.8.2 // indirect
 	github.com/tidwall/btree v1.7.0 // indirect
 	github.com/tidwall/grect v0.1.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
@@ -192,17 +194,16 @@ require (
 	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/net v0.31.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/term v0.26.0 // indirect
+	golang.org/x/term v0.28.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gorm.io/plugin/dbresolver v1.5.3 // indirect
-	modernc.org/libc v1.37.6 // indirect
-	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/memory v1.7.2 // indirect
-	modernc.org/sqlite v1.28.0 // indirect
+	modernc.org/libc v1.22.5 // indirect
+	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.5.0 // indirect
+	modernc.org/sqlite v1.23.1 // indirect
 )
 
 replace (