fix: 移植原生batch逻辑，以获取更好的内存速度

dice/dice_help.go

@@ -24,9 +24,6 @@ import (
 	"github.com/xuri/excelize/v2"
 )
 
-// 分词器封存了，看起来不太需要
-// _ "github.com/leopku/bleve-gse-tokenizer/v2"
-
 const HelpBuiltinGroup = "builtin"
 
 const (
@@ -85,11 +82,6 @@ type HelpConfig struct {
 	Aliases map[string][]string `yaml:"aliases" json:"aliases"`
 }
 
-func (m *HelpManager) GetNextID() string {
-	m.CurID++
-	return strconv.FormatUint(m.CurID, 10)
-}
-
 type HelpDocFormat struct {
 	Mod     string            `json:"mod"`
 	Author  string            `json:"author"`
@@ -230,7 +222,7 @@ func (m *HelpManager) Load() {
 	_ = m.AddItemApply(true)
 	m.CurID = m.searchEngine.GetTotalID()
 	elapsed := time.Since(start) // 计算执行时间
-	log.Infof("帮助文档加载完毕，共耗费时间: %s\n", elapsed)
+	log.Infof("帮助文档加载完毕，共耗费时间: %s 共计加载条目:%d\n", elapsed, m.CurID)
 }
 
 func (m *HelpManager) loadHelpConfig() {
@@ -465,7 +457,6 @@ func (m *HelpManager) Search(ctx *MsgContext, text string, titleOnly bool, pageS
 	return m.searchEngine.Search(ctx.Group.HelpPackages, text, titleOnly, pageSize, pageNum, group)
 }
 
-// TODO: 这些到时候写在Engine里
 func (m *HelpManager) GetSuffixText() string {
 	return m.searchEngine.GetSuffixText()
 }

dice/docengine/bleve.go

@@ -15,9 +15,10 @@ import (
 )
 
 type BleveSearchEngine struct {
-	Index bleve.Index
-	batch *bleve.Batch
-	CurID uint64
+	Index     bleve.Index
+	batch     *bleve.Batch
+	batchSize int
+	CurID     uint64
 }
 
 var indexDir = "./data/_index"
@@ -58,10 +59,10 @@ func (d *BleveSearchEngine) Init() error {
 	contentFieldMapping := bleve.NewTextFieldMapping()
 	keywordMapping := bleve.NewKeywordFieldMapping()
 	// 注意： 这里group,from,title，package都是keywordMapping，这样就能进行精确搜索。
-	// 文本才是真正的文档
 	docMapping.AddFieldMappingsAt("group", keywordMapping)
 	docMapping.AddFieldMappingsAt("from", keywordMapping)
-	docMapping.AddFieldMappingsAt("title", keywordMapping)
+	docMapping.AddFieldMappingsAt("title", contentFieldMapping)
+	// Content才是真正的文档
 	docMapping.AddFieldMappingsAt("content", contentFieldMapping)
 	docMapping.AddFieldMappingsAt("package", keywordMapping)
 	mapping.AddDocumentMapping("helpdoc", docMapping)
@@ -71,6 +72,8 @@ func (d *BleveSearchEngine) Init() error {
 		return err
 	}
 	d.Index = i
+	// 初始化ID列表
+	d.CurID = 0
 	// 初始化新的batch
 	d.batch = d.Index.NewBatch()
 	return nil
@@ -102,6 +105,15 @@ func (d *BleveSearchEngine) AddItem(item HelpTextItem) (string, error) {
 		"package": item.PackageName,
 		"_type":   "helpdoc",
 	}
+	d.batchSize++
+	// 五十一次执行
+	if d.batchSize >= 50 {
+		err := d.AddItemApply(false)
+		d.batchSize = 0
+		if err != nil {
+			return "", err
+		}
+	}
 	return id, d.batch.Index(id, data)
 }
 
@@ -275,6 +287,7 @@ func (d *BleveSearchEngine) GetItemByID(id string) (*HelpTextItem, error) {
 			item.Content = value
 		case "package":
 			item.PackageName = value
+			// 好像会碰到Type的参数？
 		default:
 			log.Debugf("这是个什么参数 %s", name)
 		}

dice/docengine/enter.go

@@ -49,6 +49,6 @@ type SearchEngine interface {
 	GetItemByID(id string) (*HelpTextItem, error)
 	// PaginateDocuments 分页获取数据
 	PaginateDocuments(pageSize, pageNum int, group, from, title string) (uint64, []*HelpTextItem, error)
-	// GetTotalID()
+	// GetTotalID 获取当前ID总数，注意，ID必须是顺序排列的
 	GetTotalID() uint64
 }

go.mod

@@ -83,7 +83,6 @@ require (
 	github.com/blevesearch/bleve_index_api v1.1.13
 	github.com/go-gorm/caches/v4 v4.0.5
 	github.com/gofrs/flock v0.12.1
-	github.com/klauspost/compress v1.17.11
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/tidwall/gjson v1.17.0
 	github.com/tidwall/sjson v1.2.5
@@ -197,6 +196,7 @@ replace (
 	github.com/Szzrain/dodo-open-go v0.2.7 => github.com/sealdice/dodo-open-go v0.2.8
 	// Try to fix arm64 bug with better snappy.
 	github.com/blevesearch/zapx/v16 v16.1.8 => github.com/PaienNate/zapx/v16 v16.1.9
+	// Try to fix sqlite in cgofree
 	github.com/glebarez/sqlite v1.11.0 => github.com/PaienNate/sqlite v0.0.0-20241102151933-067d82f14685
 	github.com/lonelyevil/kook v0.0.31 => github.com/sealdice/kook v0.0.3
 	github.com/sacOO7/gowebsocket v0.0.0-20221109081133-70ac927be105 => github.com/fy0/GoWebsocket v0.0.0-20231128163937-aa5c110b25c6

go.sum

@@ -229,8 +229,6 @@ github.com/juliangruber/go-intersect v1.1.0/go.mod h1:WMau+1kAmnlQnKiikekNJbtGtf
 github.com/kardianos/service v1.2.2 h1:ZvePhAHfvo0A7Mftk/tEzqEZ7Q4lgnR8sGz4xu1YX60=
 github.com/kardianos/service v1.2.2/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=