From af329e37f20d0e4379e17266fad322659b9b96d2 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Thu, 13 Apr 2023 12:13:38 +0800 Subject: [PATCH 01/47] improve LevelDb (UserDb ) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UserDb ( 支持兩種格式 userdb ,userdb.txt) --- src/types_ext.cc | 63 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/src/types_ext.cc b/src/types_ext.cc index 330fa7e..fd9159a 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include "lib/lua_export_type.h" #include "lib/luatype_boost_optional.h" @@ -236,44 +236,71 @@ namespace DbAccessorReg{ { NULL, NULL }, }; } -namespace LevelDbReg{ - typedef LevelDb T; +namespace UserDbReg{ + typedef Db T; typedef DbAccessor A; - // - an make(const string& file_name, const string& db_name){ - return New(file_name,db_name,"userdb"); + an make(const string& db_name, const string& db_class) { + if (auto comp= UserDb::Require(db_class)){ + return an( comp->Create(db_name)) ; + } + else { + return {}; + } + } + + an make_leveldb(const string& db_name) { + return make(db_name, "userdb"); + } + + an make_tabledb(const string& db_name) { + return make(db_name, "plain_userdb"); } - optional fetch(an t, const string& key){ + + optional fetch(an t, const string& key) { string res; if ( t->Fetch(key,&res) ) return res; return {}; } - bool loaded(an t){ - return t->loaded(); + bool Open(T &t) { return t.Open(); } + bool Close(T &t) { return t.Close(); } + bool OpenReadOnly(T &t) { return t.OpenReadOnly(); } + bool Erase(T &t, const string &key) { return t.Erase(key); } + bool Update(T &t, const string &key, const string &value) { + return t.Update(key, value); } + an Query(T &t, const string& key) { return t.Query(key); } + static const luaL_Reg funcs[] = { - {"LevelDb", WRAP(make)}, + {"UserDb", WRAP(make)},// an LevelDb( db_file, db_name) + {"LevelDb", WRAP(make_leveldb)},// an LevelDb( db_file, db_name) + {"TableDb", WRAP(make_tabledb)},// Db UserDb( db_name, db_type:userdb|plain_userdb) { NULL, NULL }, }; static const luaL_Reg methods[] = { - {"open", WRAPMEM(T::Open)}, - {"open_read_only", WRAPMEM(T::OpenReadOnly)}, - {"close", WRAPMEM(T::Close)}, - {"query", WRAPMEM(T::Query)}, // query(prefix_key) return DbAccessor + {"open", WRAP(Open)}, + {"open_read_only", WRAP(OpenReadOnly)}, + {"close", WRAP(Close)}, + {"query", WRAP(Query)}, // query(prefix_key) return DbAccessor {"fetch", WRAP(fetch)}, // fetch(key) return value - {"update", WRAPMEM(T::Update)}, // update(key,value) return bool - {"erase", WRAPMEM(T::Erase)}, // erase(key) return bool - {"loaded",WRAPMEM(T,loaded)}, + {"update", WRAP(Update)}, // update(key,value) return bool + {"erase", WRAP(Erase)}, // erase(key) return bool + {"disable", WRAPMEM(T, disable)}, + {"enable", WRAPMEM(T, enable)}, { NULL, NULL }, }; static const luaL_Reg vars_get[] = { + {"loaded",WRAPMEM(T, loaded)}, + {"read_only",WRAPMEM(T, readonly)}, + {"disabled",WRAPMEM(T, disabled)}, + {"name", WRAPMEM(T, name)}, + {"file_name", WRAPMEM(T, file_name)}, { NULL, NULL }, }; @@ -338,6 +365,6 @@ void LUAWRAPPER_LOCAL types_ext_init(lua_State *L) { EXPORT(FilterReg, L); EXPORT(ReverseLookupDictionaryReg, L); EXPORT(DbAccessorReg, L); - EXPORT(LevelDbReg, L); + EXPORT(UserDbReg, L); ComponentReg::init(L); } From 5baf3075ee144f759d23f0d9ccced4df9976839c Mon Sep 17 00:00:00 2001 From: groverlynn Date: Sat, 20 May 2023 06:42:40 +0200 Subject: [PATCH 02/47] Update number.lua --- sample/lua/number.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sample/lua/number.lua b/sample/lua/number.lua index 126bae4..c5f4eba 100644 --- a/sample/lua/number.lua +++ b/sample/lua/number.lua @@ -11,7 +11,7 @@ local confs = { }, { comment = " 小写", - number = { [0] = "零", "一", "二", "三", "四", "五", "六", "七", "八", "九" }, + number = { [0] = "〇", "一", "二", "三", "四", "五", "六", "七", "八", "九" }, suffix = { [0] = "", "十", "百", "千" }, suffix2 = { [0] = "", "万", "亿", "万亿", "亿亿" } }, @@ -19,13 +19,13 @@ local confs = { comment = " 大寫", number = { [0] = "零", "壹", "貳", "參", "肆", "伍", "陸", "柒", "捌", "玖" }, suffix = { [0] = "", "拾", "佰", "仟" }, - suffix2 = { [0] = "", "萬", "億", "萬億", "億億" } + suffix2 = { [0] = "", "萬", "億", "兆", "京" } }, { comment = " 小寫", - number = { [0] = "零", "一", "二", "三", "四", "五", "六", "七", "八", "九" }, + number = { [0] = "〇", "一", "二", "三", "四", "五", "六", "七", "八", "九" }, suffix = { [0] = "", "十", "百", "千" }, - suffix2 = { [0] = "", "萬", "億", "萬億", "億億" } + suffix2 = { [0] = "", "萬", "億", "兆", "京" } }, } From ede658fa2ff975cb280846ab0c1ec6943aa9d797 Mon Sep 17 00:00:00 2001 From: groverlynn Date: Sat, 20 May 2023 06:52:14 +0200 Subject: [PATCH 03/47] Update charset.lua --- sample/lua/charset.lua | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/sample/lua/charset.lua b/sample/lua/charset.lua index 081ddcc..71c9694 100644 --- a/sample/lua/charset.lua +++ b/sample/lua/charset.lua @@ -17,15 +17,16 @@ -- 帮助函数（可跳过） local charset = { - ["CJK"] = { first = 0x4E00, last = 0x9FFF }, -- CJK Unified Ideographs - https://unicode.org/charts/PDF/U4E00.pdf - ["ExtA"] = { first = 0x3400, last = 0x4DBF }, -- CJK Unified Ideographs Extension A - https://unicode.org/charts/PDF/U3400.pdf - ["ExtB"] = { first = 0x20000, last = 0x2A6DF }, -- CJK Unified Ideographs Extension B - https://unicode.org/charts/PDF/U20000.pdf - ["ExtC"] = { first = 0x2A700, last = 0x2B73F }, -- CJK Unified Ideographs Extension C - https://unicode.org/charts/PDF/U2A700.pdf - ["ExtD"] = { first = 0x2B740, last = 0x2B81F }, -- CJK Unified Ideographs Extension D - https://unicode.org/charts/PDF/U2B740.pdf - ["ExtE"] = { first = 0x2B820, last = 0x2CEAF }, -- CJK Unified Ideographs Extension E - https://unicode.org/charts/PDF/U2B820.pdf - ["ExtF"] = { first = 0x2CEB0, last = 0x2EBEF }, -- CJK Unified Ideographs Extension F - https://unicode.org/charts/PDF/U2CEB0.pdf - ["ExtG"] = { first = 0x30000, last = 0x3134A }, -- CJK Unified Ideographs Extension G - https://unicode.org/charts/PDF/U30000.pdf - ["Compat"] = { first = 0x2F800, last = 0x2FA1F } -- CJK Compatibility Ideographs Supplement - https://unicode.org/charts/PDF/U2F800.pdf + ["CJK"] = { first = 0x4E00, last = 0x9FFF }, -- CJK Unified Ideographs - https://unicode.org/charts/PDF/U4E00.pdf + ["ExtA"] = { first = 0x3400, last = 0x4DBF }, -- CJK Unified Ideographs Extension A - https://unicode.org/charts/PDF/U3400.pdf + ["ExtB"] = { first = 0x20000, last = 0x2A6DF }, -- CJK Unified Ideographs Extension B - https://unicode.org/charts/PDF/U20000.pdf + ["ExtC"] = { first = 0x2A700, last = 0x2B73F }, -- CJK Unified Ideographs Extension C - https://unicode.org/charts/PDF/U2A700.pdf + ["ExtD"] = { first = 0x2B740, last = 0x2B81F }, -- CJK Unified Ideographs Extension D - https://unicode.org/charts/PDF/U2B740.pdf + ["ExtE"] = { first = 0x2B820, last = 0x2CEAF }, -- CJK Unified Ideographs Extension E - https://unicode.org/charts/PDF/U2B820.pdf + ["ExtF"] = { first = 0x2CEB0, last = 0x2EBEF }, -- CJK Unified Ideographs Extension F - https://unicode.org/charts/PDF/U2CEB0.pdf + ["ExtG"] = { first = 0x30000, last = 0x3134A }, -- CJK Unified Ideographs Extension G - https://unicode.org/charts/PDF/U30000.pdf + ["Compat"] = { first = 0xF900, last = 0xFAFF }, -- CJK Compatibility Ideographs - https://unicode.org/charts/PDF/UF900.pdf + ["CompatSupp"] = { first = 0x2F800, last = 0x2FA1F } -- CJK Compatibility Ideographs Supplement - https://unicode.org/charts/PDF/U2F800.pdf } local function exists(single_filter, text) @@ -48,7 +49,8 @@ local function is_cjk_ext(c) return is_charset("ExtA")(c) or is_charset("ExtB")(c) or is_charset("ExtC")(c) or is_charset("ExtD")(c) or is_charset("ExtE")(c) or is_charset("ExtF")(c) or - is_charset("ExtG")(c) or is_charset("Compat")(c) + is_charset("ExtG")(c) or is_charset("Compat")(c) or + is_charset("CompatSupp") end --[[ From d7e740ef7900667ac67e27b49043a93355277118 Mon Sep 17 00:00:00 2001 From: groverlynn Date: Mon, 22 May 2023 04:51:23 +0200 Subject: [PATCH 04/47] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=BA=8F=E6=95=B8?= =?UTF-8?q?=E8=A9=9E=EF=BC=8C=E4=BF=AE=E6=AD=A3=E5=8D=81=EF=BD=9E=E5=8D=81?= =?UTF-8?q?=E4=B9=9D=E7=9A=84=E5=A4=A7=E5=B0=8F=E5=AF=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sample/lua/number.lua | 144 +++++++++++++++++++++++------------------- 1 file changed, 79 insertions(+), 65 deletions(-) diff --git a/sample/lua/number.lua b/sample/lua/number.lua index c5f4eba..6b4ac47 100644 --- a/sample/lua/number.lua +++ b/sample/lua/number.lua @@ -3,90 +3,104 @@ number_translator: 将 `/` + 阿拉伯数字 翻译为大小写汉字 --]] local confs = { - { - comment = " 大写", - number = { [0] = "零", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖" }, - suffix = { [0] = "", "拾", "佰", "仟" }, - suffix2 = { [0] = "", "万", "亿", "万亿", "亿亿" } - }, - { - comment = " 小写", - number = { [0] = "〇", "一", "二", "三", "四", "五", "六", "七", "八", "九" }, - suffix = { [0] = "", "十", "百", "千" }, - suffix2 = { [0] = "", "万", "亿", "万亿", "亿亿" } - }, - { - comment = " 大寫", - number = { [0] = "零", "壹", "貳", "參", "肆", "伍", "陸", "柒", "捌", "玖" }, - suffix = { [0] = "", "拾", "佰", "仟" }, - suffix2 = { [0] = "", "萬", "億", "兆", "京" } - }, - { - comment = " 小寫", - number = { [0] = "〇", "一", "二", "三", "四", "五", "六", "七", "八", "九" }, - suffix = { [0] = "", "十", "百", "千" }, - suffix2 = { [0] = "", "萬", "億", "兆", "京" } - }, + { + comment = " 大写", + numeral = { [0] = "零", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖" }, + place = { [0] = "", "拾", "佰", "仟" }, + group = { [0] = "", "万", "亿", "万亿", "亿亿" } + }, + { + comment = " 小写", + numeral = { [0] = "零", "一", "二", "三", "四", "五", "六", "七", "八", "九" }, + place = { [0] = "", "十", "百", "千" }, + group = { [0] = "", "万", "亿", "万亿", "亿亿" } + }, + { + comment = " 序数", + numeral = { [0] = "〇", "一", "二", "三", "四", "五", "六", "七", "八", "九" } + }, + { + comment = " 大寫", + numeral = { [0] = "零", "壹", "貳", "參", "肆", "伍", "陸", "柒", "捌", "玖" }, + place = { [0] = "", "拾", "佰", "仟" }, + group = { [0] = "", "萬", "億", "兆", "京" } + }, + { + comment = " 小寫", + numeral = { [0] = "零", "一", "二", "三", "四", "五", "六", "七", "八", "九" }, + place = { [0] = "", "十", "百", "千" }, + group = { [0] = "", "萬", "億", "兆", "京" } + }, + { + comment = " 序數", + numeral = { [0] = "〇", "一", "二", "三", "四", "五", "六", "七", "八", "九" } + } } local function read_seg(conf, n) - local s = "" - local i = 0 - local zf = true + local s = "" + local i = 0 + local zf = true - while string.len(n) > 0 do - local d = tonumber(string.sub(n, -1, -1)) - if d ~= 0 then - s = conf.number[d] .. conf.suffix[i] .. s - zf = false - else - if not zf then - s = conf.number[0] .. s - end - zf = true + while string.len(n) > 0 do + local d = tonumber(string.sub(n, -1, -1)) + if conf.place == nil then + s = conf.numeral[d] .. s + elseif d == 1 and i == 1 then + s = conf.place[i] .. s + elseif d ~= 0 then + s = conf.numeral[d] .. conf.place[i] .. s + zf = false + else + if not zf then + s = conf.numeral[0] .. s end - i = i + 1 - n = string.sub(n, 1, -2) - end + zf = true + end + i = i + 1 + n = string.sub(n, 1, -2) + end - return i < 4, s + return i < 4, s end local function read_number(conf, n) - local s = "" - local i = 0 - local zf = false + local s = "" + local i = 0 + local zf = false - n = string.gsub(n, "^0+", "") + n = string.gsub(n, "^0+", "") - if n == "" then - return conf.number[0] - end + if n == "" then + return conf.numeral[0] + end - while string.len(n) > 0 do - local zf2, r = read_seg(conf, string.sub(n, -4, -1)) - if r ~= "" then - if zf and s ~= "" then - s = r .. conf.suffix2[i] .. conf.number[0] .. s - else - s = r .. conf.suffix2[i] .. s - end + while string.len(n) > 0 do + local zf2, r = read_seg(conf, string.sub(n, -4, -1)) + if r ~= "" then + if conf.group == nil then + s = r .. s + elseif zf and s ~= "" then + s = r .. conf.group[i] .. conf.numeral[0] .. s + else + s = r .. conf.group[i] .. s end - zf = zf2 - i = i + 1 - n = string.sub(n, 1, -5) - end - return s + end + zf = zf2 + i = i + 1 + n = string.sub(n, 1, -5) + end + return s end local function translator(input, seg) if string.sub(input, 1, 1) == "/" then local n = string.sub(input, 2) if tonumber(n) ~= nil then - for _, conf in ipairs(confs) do - local r = read_number(conf, n) - yield(Candidate("number", seg.start, seg._end, r, conf.comment)) - end + for _, conf in ipairs(confs) do + local r = read_number(conf, n) + yield(Candidate("number", seg.start, seg._end, r, conf.comment)) + end end end end From 30c83ee7dfa7224814fb62acf72faeded9efebb4 Mon Sep 17 00:00:00 2001 From: groverlynn Date: Mon, 22 May 2023 04:59:33 +0200 Subject: [PATCH 05/47] Update number.lua --- sample/lua/number.lua | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sample/lua/number.lua b/sample/lua/number.lua index 6b4ac47..acfb679 100644 --- a/sample/lua/number.lua +++ b/sample/lua/number.lua @@ -94,14 +94,14 @@ local function read_number(conf, n) end local function translator(input, seg) - if string.sub(input, 1, 1) == "/" then - local n = string.sub(input, 2) - if tonumber(n) ~= nil then - for _, conf in ipairs(confs) do - local r = read_number(conf, n) - yield(Candidate("number", seg.start, seg._end, r, conf.comment)) - end + if string.sub(input, 1, 1) == "/" then + local n = string.sub(input, 2) + if tonumber(n) ~= nil then + for _, conf in ipairs(confs) do + local r = read_number(conf, n) + yield(Candidate("number", seg.start, seg._end, r, conf.comment)) end + end end end From 7e821c60653b3c57c667ac48a44cd8bdfb888fb0 Mon Sep 17 00:00:00 2001 From: groverlynn Date: Mon, 22 May 2023 05:00:21 +0200 Subject: [PATCH 06/47] Update number.lua --- sample/lua/number.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample/lua/number.lua b/sample/lua/number.lua index acfb679..e47a308 100644 --- a/sample/lua/number.lua +++ b/sample/lua/number.lua @@ -102,7 +102,7 @@ local function translator(input, seg) yield(Candidate("number", seg.start, seg._end, r, conf.comment)) end end - end + end end return translator From 5d7f99444b17ff12b6955ea26c434d7022a0e8fd Mon Sep 17 00:00:00 2001 From: groverlynn Date: Fri, 23 Jun 2023 16:50:41 +0200 Subject: [PATCH 07/47] Update number.lua --- sample/lua/number.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample/lua/number.lua b/sample/lua/number.lua index e47a308..dd3a1ac 100644 --- a/sample/lua/number.lua +++ b/sample/lua/number.lua @@ -46,7 +46,7 @@ local function read_seg(conf, n) local d = tonumber(string.sub(n, -1, -1)) if conf.place == nil then s = conf.numeral[d] .. s - elseif d == 1 and i == 1 then + elseif d == 1 and string.len(n) == 1 then s = conf.place[i] .. s elseif d ~= 0 then s = conf.numeral[d] .. conf.place[i] .. s From ea1c087d3ef5bdf9b564b829987948146547cd80 Mon Sep 17 00:00:00 2001 From: groverlynn Date: Fri, 23 Jun 2023 16:55:32 +0200 Subject: [PATCH 08/47] Update number.lua --- sample/lua/number.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample/lua/number.lua b/sample/lua/number.lua index dd3a1ac..768626a 100644 --- a/sample/lua/number.lua +++ b/sample/lua/number.lua @@ -46,7 +46,7 @@ local function read_seg(conf, n) local d = tonumber(string.sub(n, -1, -1)) if conf.place == nil then s = conf.numeral[d] .. s - elseif d == 1 and string.len(n) == 1 then + elseif d == 1 and i == 1 and string.len(n) == 1 then s = conf.place[i] .. s elseif d ~= 0 then s = conf.numeral[d] .. conf.place[i] .. s From 67758806b78bd242f55b9b2b76f4eabe8b6cef56 Mon Sep 17 00:00:00 2001 From: Chunhui He Date: Sat, 24 Jun 2023 03:37:09 +0000 Subject: [PATCH 09/47] fix #249 --- sample/lua/charset.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample/lua/charset.lua b/sample/lua/charset.lua index 71c9694..77998b9 100644 --- a/sample/lua/charset.lua +++ b/sample/lua/charset.lua @@ -50,7 +50,7 @@ local function is_cjk_ext(c) is_charset("ExtC")(c) or is_charset("ExtD")(c) or is_charset("ExtE")(c) or is_charset("ExtF")(c) or is_charset("ExtG")(c) or is_charset("Compat")(c) or - is_charset("CompatSupp") + is_charset("CompatSupp")(c) end --[[ From 6272ce5a49b0c77ae7094133cb893bcc8e6ccfc9 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Sat, 24 Jun 2023 17:01:04 +0800 Subject: [PATCH 10/47] Update types_ext.cc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit loaded() 移回 methods 增加 vars_get _loaded --- src/types_ext.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/types_ext.cc b/src/types_ext.cc index fd9159a..fce79fe 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -289,14 +289,15 @@ namespace UserDbReg{ {"fetch", WRAP(fetch)}, // fetch(key) return value {"update", WRAP(Update)}, // update(key,value) return bool {"erase", WRAP(Erase)}, // erase(key) return bool - + + {"loaded",WRAPMEM(T, loaded)}, {"disable", WRAPMEM(T, disable)}, {"enable", WRAPMEM(T, enable)}, { NULL, NULL }, }; static const luaL_Reg vars_get[] = { - {"loaded",WRAPMEM(T, loaded)}, + {"_loaded",WRAPMEM(T, loaded)}, {"read_only",WRAPMEM(T, readonly)}, {"disabled",WRAPMEM(T, disabled)}, {"name", WRAPMEM(T, name)}, From c985eb399d63c491c6e03fa1651ddb59e485c04a Mon Sep 17 00:00:00 2001 From: Chunhui He Date: Sun, 25 Jun 2023 13:00:59 +0000 Subject: [PATCH 11/47] refactor(lib): reduce binary size --- src/lib/lua.cc | 17 +++++++++++++++++ src/lib/lua.h | 4 ++++ src/lib/lua_templates.h | 15 +-------------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/lib/lua.cc b/src/lib/lua.cc index fab99bf..c68fdc1 100644 --- a/src/lib/lua.cc +++ b/src/lib/lua.cc @@ -2,6 +2,23 @@ #include "lua_templates.h" namespace LuaImpl { + int wrap_common(lua_State *L, int (*cfunc)(lua_State *)) { + char room[sizeof(C_State)]; + C_State *C = new (&room) C_State(); + lua_pushcfunction(L, cfunc); + lua_insert(L, 1); + lua_pushlightuserdata(L, (void *) C); + lua_insert(L, 2); + int status = lua_pcall(L, lua_gettop(L) - 1, LUA_MULTRET, 0); + if (status != LUA_OK) { + C->~C_State(); + lua_error(L); + abort(); // unreachable + } + C->~C_State(); + return lua_gettop(L); + } + static int index(lua_State *L) { if (luaL_getmetafield(L, 1, "methods") != LUA_TNIL) { lua_pushvalue(L, 2); diff --git a/src/lib/lua.h b/src/lib/lua.h index 29e0633..6eedd87 100644 --- a/src/lib/lua.h +++ b/src/lib/lua.h @@ -53,4 +53,8 @@ class Lua { lua_State *L_; }; +namespace LuaImpl { + int wrap_common(lua_State *L, int (*cfunc)(lua_State *)); +} + #endif // LIB_LUA_H_ diff --git a/src/lib/lua_templates.h b/src/lib/lua_templates.h index a98a577..41c3ca6 100644 --- a/src/lib/lua_templates.h +++ b/src/lib/lua_templates.h @@ -595,20 +595,7 @@ struct LuaWrapper { } static int wrap(lua_State *L) { - char room[sizeof(C_State)]; - C_State *C = new (&room) C_State(); - lua_pushcfunction(L, wrap_helper); - lua_insert(L, 1); - lua_pushlightuserdata(L, (void *) C); - lua_insert(L, 2); - int status = lua_pcall(L, lua_gettop(L) - 1, LUA_MULTRET, 0); - if (status != LUA_OK) { - C->~C_State(); - lua_error(L); - abort(); // unreachable - } - C->~C_State(); - return lua_gettop(L); + return LuaImpl::wrap_common(L, wrap_helper); } }; From 477fb068150e996e9e2730f1ed6f7fe40a48f8b3 Mon Sep 17 00:00:00 2001 From: shewer Date: Sat, 9 Sep 2023 19:11:50 +0800 Subject: [PATCH 12/47] add notifier:connect(func,int: group) Signed-off-by: shewer --- src/types.cc | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/types.cc b/src/types.cc index a8b73c4..b2e1477 100644 --- a/src/types.cc +++ b/src/types.cc @@ -1208,16 +1208,15 @@ static int raw_connect(lua_State *L) { Lua *lua = Lua::from_state(L); T & t = LuaType::todata(L, 1); an o = LuaObj::todata(L, 2); + auto f = [lua, o](I... i) { + auto r = lua->void_call, Context *>(o, i...); + if (!r.ok()) { + auto e = r.get_err(); + LOG(ERROR) << "Context::Notifier error(" << e.status << "): " << e.e; + } + }; - auto c = t.connect - ([lua, o](I... i) { - auto r = lua->void_call, Context *>(o, i...); - if (!r.ok()) { - auto e = r.get_err(); - LOG(ERROR) << "Context::Notifier error(" << e.status << "): " << e.e; - } - }); - + auto c = (lua_gettop(L) > 2) ? t.connect(lua_tointeger(L, 3), f) : t.connect(f); LuaType::pushdata(L, c); return 1; } From a06b97396523c3424310435f8751b6a7f8ca4507 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Fri, 15 Sep 2023 19:39:44 +0800 Subject: [PATCH 13/47] Update types.cc improve Projection:apply function (#269) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update types.cc improve Projection:apply function add args bool ret_org_str apply(string, bool = false) apply(string) 仍然延續 轉換失敗時 return "" apply(string,true) 轉換失敗時 return 原 字串 * Update types.cc improve Projection:apply function Signed-off-by: shewer * replace WARN(apply) to raw_apply Signed-off-by: shewer --------- Signed-off-by: shewer --- src/types.cc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/types.cc b/src/types.cc index b2e1477..f0815ba 100644 --- a/src/types.cc +++ b/src/types.cc @@ -1058,12 +1058,15 @@ namespace ProjectionReg{ return New(); } - string apply(T &t, const string &s){ - string res= s; - if (t.Apply(&res)) - return res; - else - return ""; + int raw_apply(lua_State* L) { + an t = LuaType>::todata(L, 1); + string res(lua_tostring(L, 2)); + bool ret_org_str = lua_gettop(L)>2 && lua_toboolean(L, 3); + if (!t->Apply(&res) && !ret_org_str) + res.clear(); + + LuaType::pushdata(L, res); + return 1; } static const luaL_Reg funcs[] = { @@ -1073,7 +1076,7 @@ namespace ProjectionReg{ static const luaL_Reg methods[] = { {"load",WRAPMEM(T::Load)}, - {"apply",WRAP(apply)}, + {"apply", raw_apply}, { NULL, NULL }, }; From 6773dea11818d04ae3c0c6187194453dd1accb9d Mon Sep 17 00:00:00 2001 From: fxliang Date: Sun, 17 Sep 2023 10:23:04 +0800 Subject: [PATCH 14/47] feat: autoload module with sub-table/sub-function defined (#257) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: autoload module with sub-table/sub-function defined, example *module_name*subtable1*subtable2*targetfunc * Update lua_gears.cc 改善 檢查 submodule 流程 ，最後一個 可以是 function @*module*sub1*sub2*sub3 for ( i=1; i <= size-1; i++) sub1 table i=1 sub2 table i=2 sub3 table or function i= size-1 * code moved to sub_module_init() Signed-off-by: shewer * code moved to sub_module_init() Signed-off-by: shewer * sub_module_init() :fix index count error Signed-off-by: shewer * sub_module_init() push error msg ：可避免 function 進入init 程序 Signed-off-by: shewer --------- Signed-off-by: shewer Co-authored-by: Shewer Lu --- src/lua_gears.cc | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/src/lua_gears.cc b/src/lua_gears.cc index beb0fdc..2b56238 100644 --- a/src/lua_gears.cc +++ b/src/lua_gears.cc @@ -1,5 +1,7 @@ #include "lib/lua_templates.h" #include "lua_gears.h" +#include +#include namespace rime { @@ -21,6 +23,39 @@ bool LuaTranslation::Next() { } } +static std::vector split_string(const std::string& str, const std::string& delimiter) { + std::vector result; + size_t pos = 0; + size_t found; + while ((found = str.find(delimiter, pos)) != std::string::npos) { + result.push_back(str.substr(pos, found - pos)); + pos = found + delimiter.length(); + } + result.push_back(str.substr(pos)); + return result; +} +static bool sub_module_init(lua_State *L, const Ticket &t, + const std::vector& vec_klass) { + size_t vec_klass_sz= vec_klass.size(); + for (size_t index=1 ;index < vec_klass_sz; index++) { + int sub_type= lua_getfield(L, -1, vec_klass.at(index).c_str() ); + if ( index < vec_klass_sz-1 && sub_type != LUA_TTABLE ) { + std::ostringstream ostr; + ostr << "Lua Compoment of initialize error:(" + << " klass: " << t.klass + << " module: "<< vec_klass.at(0) + << ", name_space: " << t.name_space + << ", sub-table(" <::pushdata(L, ostr.str()); + return false; + } + } + return true; +} //--- static void raw_init(lua_State *L, const Ticket &t, an *env, an *func, an *fini, an *tags_match= NULL) { @@ -33,9 +68,11 @@ static void raw_init(lua_State *L, const Ticket &t, *env = LuaObj::todata(L, -1); lua_pop(L, 1); + std::vector _vec_klass = (t.klass[0] == '*') ? + split_string(t.klass.substr(1), "*") : split_string(t.klass, "*"); if (t.klass.size() > 0 && t.klass[0] == '*') { lua_getglobal(L, "require"); - lua_pushstring(L, t.klass.c_str() + 1); + lua_pushstring(L, _vec_klass.at(0).c_str()); int status = lua_pcall(L, 1, 1, 0); if (status != LUA_OK) { const char *e = lua_tostring(L, -1); @@ -46,7 +83,11 @@ static void raw_init(lua_State *L, const Ticket &t, << " ): " << e; } } else { - lua_getglobal(L, t.klass.c_str()); + lua_getglobal(L, _vec_klass.at(0).c_str()); + } + + if (_vec_klass.size() > 1) { + sub_module_init(L, t, _vec_klass); } if (lua_type(L, -1) == LUA_TTABLE) { @@ -70,7 +111,7 @@ static void raw_init(lua_State *L, const Ticket &t, *fini = LuaObj::todata(L, -1); } lua_pop(L, 1); - + if (tags_match) { lua_getfield(L, -1, "tags_match"); if (lua_type(L, -1) == LUA_TFUNCTION) { From 93230b0b4bbc7c86e9eaa551658628f92b8c887f Mon Sep 17 00:00:00 2001 From: KNnut <9387720+KNnut@users.noreply.github.com> Date: Sun, 17 Sep 2023 14:27:25 +0800 Subject: [PATCH 15/47] fix build for lua51 lua52 luajit lua_getfield returns void when lua version < 5.3 --- src/lua_gears.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lua_gears.cc b/src/lua_gears.cc index 2b56238..4e70092 100644 --- a/src/lua_gears.cc +++ b/src/lua_gears.cc @@ -38,8 +38,8 @@ static bool sub_module_init(lua_State *L, const Ticket &t, const std::vector& vec_klass) { size_t vec_klass_sz= vec_klass.size(); for (size_t index=1 ;index < vec_klass_sz; index++) { - int sub_type= lua_getfield(L, -1, vec_klass.at(index).c_str() ); - if ( index < vec_klass_sz-1 && sub_type != LUA_TTABLE ) { + lua_getfield(L, -1, vec_klass.at(index).c_str() ); + if ( index < vec_klass_sz-1 && lua_type(L, -1) != LUA_TTABLE ) { std::ostringstream ostr; ostr << "Lua Compoment of initialize error:(" << " klass: " << t.klass From 94e895fe8927e8a8b6e52add7ff67f14c25b7358 Mon Sep 17 00:00:00 2001 From: WhiredPlanck Date: Tue, 3 Oct 2023 10:06:11 +0800 Subject: [PATCH 16/47] refactor(types,type_ext): replace typedef with using --- src/types.cc | 79 ++++++++++++++++++++++++------------------------ src/types_ext.cc | 26 ++++++++-------- 2 files changed, 52 insertions(+), 53 deletions(-) diff --git a/src/types.cc b/src/types.cc index f0815ba..22efde0 100644 --- a/src/types.cc +++ b/src/types.cc @@ -31,7 +31,7 @@ namespace { //--- wrappers for Segment namespace SegmentReg { - typedef Segment T; + using T = Segment; T make(int start_pos, int end_pos) { return Segment(start_pos, end_pos); @@ -102,7 +102,7 @@ namespace SegmentReg { //--- wrappers for an namespace CandidateReg { - typedef Candidate T; + using T = Candidate; string dynamic_type(T &c) { if (dynamic_cast(&c)) @@ -207,7 +207,7 @@ namespace CandidateReg { //--- wrappers for an namespace TranslationReg { - typedef Translation T; + using T = Translation; int raw_make(lua_State *L) { Lua *lua = Lua::from_state(L); @@ -257,7 +257,7 @@ namespace TranslationReg { } namespace ReverseDbReg { - typedef ReverseDb T; + using T = ReverseDb; an make(const string &file) { an db = New(string(RimeGetUserDataDir()) + "/" + file); @@ -293,7 +293,7 @@ namespace ReverseDbReg { } namespace SegmentationReg { - typedef Segmentation T; + using T = Segmentation; optional back(T &t) { if (t.empty()) @@ -369,7 +369,7 @@ namespace SegmentationReg { } namespace MenuReg { - typedef Menu T; + using T = Menu; an make() { return New(); @@ -401,7 +401,7 @@ namespace MenuReg { } namespace KeyEventReg { - typedef KeyEvent T; + using T = KeyEvent; int keycode(const T &t) { return t.keycode(); @@ -445,7 +445,7 @@ namespace KeyEventReg { } namespace EngineReg { - typedef Engine T; + using T = Engine; static void apply_schema(T *engine, the &schema) { engine->ApplySchema(schema.release()); @@ -477,7 +477,7 @@ namespace EngineReg { } namespace CommitRecordReg { - typedef CommitRecord T; + using T = CommitRecord; static const luaL_Reg funcs[] = { { NULL, NULL }, @@ -502,9 +502,9 @@ namespace CommitRecordReg { } namespace CommitHistoryReg { - typedef CommitHistory T; - typedef CommitRecord CR; - typedef T::reverse_iterator R_ITER; + using T = CommitHistory; + using CR = CommitRecord; + using R_ITER = T::reverse_iterator; int raw_push(lua_State *L){ C_State C; @@ -606,7 +606,7 @@ namespace CommitHistoryReg { } namespace ContextReg { - typedef Context T; + using T = Context; Composition &get_composition(T &t) { return t.composition(); @@ -683,7 +683,7 @@ namespace ContextReg { } namespace PreeditReg { - typedef Preedit T; + using T = Preedit; static const luaL_Reg funcs[] = { { NULL, NULL }, @@ -711,7 +711,7 @@ namespace PreeditReg { } namespace CompositionReg { - typedef Composition T; + using T = Composition; Segmentation *toSegmentation(T &t) { return dynamic_cast(&t); @@ -761,7 +761,7 @@ namespace CompositionReg { } namespace SchemaReg { - typedef Schema T; + using T = Schema; the make(const string &schema_id) { return std::unique_ptr(new T(schema_id)); @@ -793,8 +793,8 @@ namespace SchemaReg { } namespace ConfigValueReg { - typedef ConfigValue T; - typedef ConfigItem E; + using T = ConfigValue; + using E = ConfigItem; // an make(){ // return New(); @@ -883,8 +883,8 @@ namespace ConfigValueReg { }; } namespace ConfigListReg { - typedef ConfigList T; - typedef ConfigItem E; + using T = ConfigList; + using E = ConfigItem; an make(){ return New(); @@ -935,8 +935,8 @@ namespace ConfigListReg { namespace ConfigMapReg { - typedef ConfigMap T; - typedef ConfigItem E; + using T = ConfigMap; + using E = ConfigItem; an make(){ return New(); @@ -999,10 +999,10 @@ namespace ConfigMapReg { } namespace ConfigItemReg { - typedef ConfigItem T; - typedef ConfigMap M; - typedef ConfigList L; - typedef ConfigValue V; + using T = ConfigItem; + using M = ConfigMap; + using L = ConfigList; + using V = ConfigValue; string type(T &t){ switch (t.type()) { @@ -1053,7 +1053,7 @@ namespace ConfigItemReg { } namespace ProjectionReg{ - typedef Projection T; + using T = Projection; an make(){ return New(); } @@ -1090,7 +1090,7 @@ namespace ProjectionReg{ } namespace ConfigReg { - typedef Config T; + using T = Config; optional get_bool(T &t, const string &path) { bool v; @@ -1225,7 +1225,7 @@ static int raw_connect(lua_State *L) { } namespace ConnectionReg { - typedef boost::signals2::connection T; + using T = boost::signals2::connection; static const luaL_Reg funcs[] = { { NULL, NULL }, @@ -1267,7 +1267,7 @@ namespace NotifierReg { } namespace OptionUpdateNotifierReg { - typedef Context::OptionUpdateNotifier T; + using T = Context::OptionUpdateNotifier; static const luaL_Reg funcs[] = { { NULL, NULL }, @@ -1288,7 +1288,7 @@ namespace OptionUpdateNotifierReg { } namespace PropertyUpdateNotifierReg { - typedef Context::PropertyUpdateNotifier T; + using T = Context::PropertyUpdateNotifier; static const luaL_Reg funcs[] = { { NULL, NULL }, @@ -1309,7 +1309,7 @@ namespace PropertyUpdateNotifierReg { } namespace KeyEventNotifierReg { - typedef Context::KeyEventNotifier T; + using T = Context::KeyEventNotifier; static const luaL_Reg funcs[] = { { NULL, NULL }, @@ -1356,7 +1356,7 @@ namespace LogReg { } } namespace CommitEntryReg { - typedef CommitEntry T; + using T = CommitEntry; vector get(const T& ce) { return ce.elements; @@ -1380,7 +1380,7 @@ namespace CommitEntryReg { }; } namespace DictEntryReg { - typedef DictEntry T; + using T = DictEntry; an make() { return an(new T()); } @@ -1419,8 +1419,7 @@ namespace DictEntryReg { }; } namespace CodeReg { - - typedef Code T; + using T = Code; an make() { return an(new Code()); @@ -1478,7 +1477,7 @@ namespace MemoryReg { uter = UserDictEntryIterator(); } }; - typedef LuaMemory T; + using T = LuaMemory; bool MemoryReg::LuaMemory::Memorize(const CommitEntry& commit_entry) { if (!memorize_callback) @@ -1602,7 +1601,7 @@ namespace MemoryReg { //--- wrappers for Phrase namespace PhraseReg { - typedef Phrase T; + using T = Phrase; an make(MemoryReg::LuaMemory& memory, const string& type, @@ -1660,7 +1659,7 @@ namespace PhraseReg { }// Phrase work with Translator namespace KeySequenceReg { - typedef KeySequence T; + using T = KeySequence; int raw_make(lua_State *L){ an t = (0(( lua_tostring(L,1) )) : New(); @@ -1785,7 +1784,7 @@ namespace RimeApiReg { } namespace SwitcherReg { - typedef Switcher T; + using T = Switcher; an make(Engine *engine) { return New(engine); diff --git a/src/types_ext.cc b/src/types_ext.cc index fce79fe..1ea234e 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -40,7 +40,7 @@ struct COMPAT().name_space())>> { }; namespace ProcessorReg{ - typedef Processor T; + using T = Processor; int process_key_event(T &t, const KeyEvent &key){ switch (t.ProcessKeyEvent(key) ){ @@ -71,7 +71,7 @@ namespace ProcessorReg{ } namespace SegmentorReg{ - typedef Segmentor T; + using T = Segmentor; bool proceed(T &t, Segmentation & s) { return t.Proceed(&s); @@ -97,7 +97,7 @@ namespace SegmentorReg{ } namespace TranslatorReg{ - typedef Translator T; + using T = Translator; static const luaL_Reg funcs[] = { { NULL, NULL }, @@ -119,7 +119,7 @@ namespace TranslatorReg{ } namespace FilterReg{ - typedef Filter T; + using T = Filter; static const luaL_Reg funcs[] = { { NULL, NULL }, @@ -142,8 +142,8 @@ namespace FilterReg{ } // ReverseDictionary namespace ReverseLookupDictionaryReg { - typedef ReverseLookupDictionary T; - typedef ReverseLookupDictionaryComponent C; + using T = ReverseLookupDictionary; + using C = ReverseLookupDictionaryComponent; an make(const string& dict_name) { if ( auto c = (C *) T::Require("reverse_lookup_dictionary")){ @@ -186,7 +186,7 @@ namespace ReverseLookupDictionaryReg { // leveldb namespace DbAccessorReg{ - typedef DbAccessor T; + using T = DbAccessor; // return key , value or nil int raw_next(lua_State* L){ @@ -237,8 +237,8 @@ namespace DbAccessorReg{ }; } namespace UserDbReg{ - typedef Db T; - typedef DbAccessor A; + using T = Db; + using A = DbAccessor; an make(const string& db_name, const string& db_class) { if (auto comp= UserDb::Require(db_class)){ @@ -311,10 +311,10 @@ namespace UserDbReg{ } namespace ComponentReg{ - typedef Processor P; - typedef Segmentor S; - typedef Translator T; - typedef Filter F; + using P = Processor; + using S = Segmentor; + using T = Translator; + using F = Filter; template int raw_create(lua_State *L){ From 7f68b898da1aac11b9b0b025bd0147b9a2ec9027 Mon Sep 17 00:00:00 2001 From: WhiredPlanck Date: Tue, 3 Oct 2023 10:50:42 +0800 Subject: [PATCH 17/47] refactor(types_ext): explicitly include cstddef and declear nullptr_t under std --- src/types_ext.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/types_ext.cc b/src/types_ext.cc index 1ea234e..97b8b17 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -5,6 +5,7 @@ * Distributed under terms of the MIT license. */ +#include #include #include #include @@ -27,7 +28,7 @@ template using void_t = void; template struct COMPAT { // fallback version of name_space() if librime is old - static nullptr_t name_space(T &t) { + static std::nullptr_t name_space(T &t) { return nullptr; } }; From aa517eb2e5a816399efbdbfb5db8002494772a53 Mon Sep 17 00:00:00 2001 From: Chunhui He Date: Wed, 4 Oct 2023 12:06:30 +0000 Subject: [PATCH 18/47] chore(lib): fix compilation errors in C++17 --- src/lib/lua_templates.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/lib/lua_templates.h b/src/lib/lua_templates.h index 41c3ca6..5612927 100644 --- a/src/lib/lua_templates.h +++ b/src/lib/lua_templates.h @@ -629,6 +629,32 @@ struct MemberWrapper { } }; +#if __cplusplus >= 201703L +template +struct MemberWrapper { + template + static R wrapT(D &c, T... t) { + return (c.*f)(t...); + } + + static R wrap(C &c, T... t) { + return wrapT(c, t...); + } +}; + +template +struct MemberWrapper { + template + static R wrapT(const D &c, T... t) { + return (c.*f)(t...); + } + + static R wrap(const C &c, T... t) { + return wrapT(c, t...); + } +}; +#endif + template struct LUAWRAPPER_LOCAL MemberWrapperV; From be8ea3b527602de40813b1313c29fde267a6cc7f Mon Sep 17 00:00:00 2001 From: Chunhui He Date: Thu, 5 Oct 2023 11:31:54 +0000 Subject: [PATCH 19/47] chore(lib): fix more MSVC/AppleClang compilation errors in C++17 --- src/lib/lua_templates.h | 2 +- src/opencc.cc | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/lua_templates.h b/src/lib/lua_templates.h index 5612927..13076db 100644 --- a/src/lib/lua_templates.h +++ b/src/lib/lua_templates.h @@ -629,7 +629,7 @@ struct MemberWrapper { } }; -#if __cplusplus >= 201703L +#if __cplusplus >= 201703L || _MSVC_LANG >= 201703L template struct MemberWrapper { template diff --git a/src/opencc.cc b/src/opencc.cc index 77f7495..4827f32 100644 --- a/src/opencc.cc +++ b/src/opencc.cc @@ -20,7 +20,9 @@ #include "lib/lua_export_type.h" #include "lib/luatype_boost_optional.h" -using namespace std; +using std::string; +using std::vector; +using std::list; using boost::optional; namespace { From 94334ca6c9d838f337bc708678b85476b7190c02 Mon Sep 17 00:00:00 2001 From: WhiredPlanck Date: Wed, 4 Oct 2023 16:03:42 +0800 Subject: [PATCH 20/47] refactor: replace boost filesystem with std filesystem --- src/opencc.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/opencc.cc b/src/opencc.cc index 4827f32..06a6be3 100644 --- a/src/opencc.cc +++ b/src/opencc.cc @@ -13,10 +13,17 @@ #include #include #include -#include #include #include +#if __cplusplus >= 201703L || _MSVC_LANG >= 201703L + #include + namespace ns = std::filesystem; +#else + #include + namespace ns = boost::filesystem; +#endif + #include "lib/lua_export_type.h" #include "lib/luatype_boost_optional.h" @@ -131,8 +138,7 @@ vector Opencc::convert_word(const string& text){ }; namespace OpenccReg { - typedef Opencc T; - namespace ns = boost::filesystem; + using T = Opencc; optional make(const string &filename) { string user_path( RimeGetUserDataDir()); From 86de0e54b2dcc84af5a02effb7f04a9b338bca70 Mon Sep 17 00:00:00 2001 From: shewer Date: Sat, 7 Oct 2023 16:25:47 +0800 Subject: [PATCH 21/47] opencc.cc: remove filesystem::exists() Signed-off-by: shewer --- src/opencc.cc | 51 ++++++++++++--------------------------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/src/opencc.cc b/src/opencc.cc index 06a6be3..4b8bf4f 100644 --- a/src/opencc.cc +++ b/src/opencc.cc @@ -13,16 +13,8 @@ #include #include #include -#include #include - -#if __cplusplus >= 201703L || _MSVC_LANG >= 201703L - #include - namespace ns = std::filesystem; -#else - #include - namespace ns = boost::filesystem; -#endif +#include #include "lib/lua_export_type.h" #include "lib/luatype_boost_optional.h" @@ -31,6 +23,7 @@ using std::string; using std::vector; using std::list; using boost::optional; +using namespace rime; namespace { @@ -51,23 +44,6 @@ class Opencc { opencc::DictPtr dict_; }; -/* -shared_ptr Opencc::create(const string &config_path) { - try { - return make_shared(config_path); - } - catch (opencc::FileNotFound &ex) { - LOG(ERROR) << config_path << " : onpecc file not found";// << ex.what(); - } - catch (opencc::InvalidFormat &ex) { - LOG(ERROR) << config_path << " : opencc file InvalidFormat";// << ex.what(); - } - catch (...){ - LOG(ERROR) << config_path << "Opencc ininialize faild" ; - } - return {}; -} -*/ Opencc::Opencc(const string& config_path) { opencc::Config config; converter_ = config.NewFromFile(config_path); @@ -143,24 +119,21 @@ namespace OpenccReg { optional make(const string &filename) { string user_path( RimeGetUserDataDir()); string shared_path(RimeGetSharedDataDir()); - user_path += "/opencc/" + filename; - shared_path += "/opencc/" + filename; - const string *path; - if (ns::exists(user_path)) - path = &user_path; - else if (ns::exists(shared_path)) - path = &shared_path; - else - path = &filename; - try{ - return T(*path); + return T(user_path + "/opencc/" + filename); } catch(...) { - LOG(ERROR) << *path << " File not found or InvalidFormat"; - return {}; + try{ + return T(shared_path + "/opencc/" + filename); + } + catch(...) { + LOG(ERROR) << " [" << user_path << "|" << shared_path << "]/opencc/" + << filename << ": File not found or InvalidFormat"; + return {}; + } } } + optional> convert_word(T &t,const string &s) { vector res; if (t.ConvertWord(s,&res)) From 1e3ab105947299b36b4e6879522299677f99ea8a Mon Sep 17 00:00:00 2001 From: Chunhui He Date: Mon, 23 Oct 2023 10:29:59 +0000 Subject: [PATCH 22/47] fix build error caused by librime's change --- src/types_ext.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/types_ext.cc b/src/types_ext.cc index 97b8b17..d66f0e9 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -20,6 +20,7 @@ #include using namespace rime; +using boost::optional; namespace { From 2faea05465cecc157f0a36bd94a8c378e50de038 Mon Sep 17 00:00:00 2001 From: Chunhui He Date: Sun, 29 Oct 2023 08:42:14 +0000 Subject: [PATCH 23/47] chore: drop use of optional referenes --- src/types.cc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/types.cc b/src/types.cc index 22efde0..8724454 100644 --- a/src/types.cc +++ b/src/types.cc @@ -295,10 +295,10 @@ namespace ReverseDbReg { namespace SegmentationReg { using T = Segmentation; - optional back(T &t) { + Segment *back(T &t) { if (t.empty()) - return {}; - return t.back(); + return nullptr; + return &t.back(); } void pop_back(T &t) { @@ -323,15 +323,15 @@ namespace SegmentationReg { return ret; } - optional get_at(T &t, const int idx) { + Segment *get_at(T &t, const int idx) { size_t size = t.size(); int index = (idx < 0) ? size + idx : idx; if (index >=0 && index < size) - return t.at(index); + return &t.at(index); LOG(WARNING) << "the index(" << idx <<")" << " is out of range(-size .. size-1); size: "<< size ; - return {}; + return nullptr; } static const luaL_Reg funcs[] = { @@ -536,10 +536,10 @@ namespace CommitHistoryReg { return 0; } - optional back(T &t) { + CR *back(T &t) { if (t.empty()) - return {}; - return t.back(); + return nullptr; + return &t.back(); } vector to_table(T &t) { @@ -717,10 +717,10 @@ namespace CompositionReg { return dynamic_cast(&t); } - optional back(T &t) { + Segment *back(T &t) { if (t.empty()) - return {}; - return t.back(); + return nullptr; + return &t.back(); } void push_back(T &t, Segment &seg) { From 7c297e4d2e08fcdd3e9b2dcae2a42317b9a217ff Mon Sep 17 00:00:00 2001 From: Chunhui He Date: Sun, 29 Oct 2023 08:13:29 +0000 Subject: [PATCH 24/47] chore: switch to std::optional when using C++17 or above --- src/lib/luatype_std_optional.h | 24 ++++++++++++++++++++++++ src/opencc.cc | 3 +-- src/optional.h | 7 +++++++ src/types.cc | 3 +-- src/types_ext.cc | 3 +-- 5 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 src/lib/luatype_std_optional.h create mode 100644 src/optional.h diff --git a/src/lib/luatype_std_optional.h b/src/lib/luatype_std_optional.h new file mode 100644 index 0000000..f65e532 --- /dev/null +++ b/src/lib/luatype_std_optional.h @@ -0,0 +1,24 @@ +#ifndef LUATYPE_STD_OPTIONAL_H +#define LUATYPE_STD_OPTIONAL_H + +#include "lua_templates.h" +#include + +template +struct LuaType> { + static void pushdata(lua_State *L, std::optional o) { + if (o) + LuaType::pushdata(L, *o); + else + lua_pushnil(L); + } + + static std::optional &todata(lua_State *L, int i, C_State *C) { + if (lua_type(L, i) == LUA_TNIL) + return C->alloc>(); + else + return C->alloc>(LuaType::todata(L, i, C)); + } +}; + +#endif /* LUATYPE_STD_OPTIONAL_H */ diff --git a/src/opencc.cc b/src/opencc.cc index 4b8bf4f..17c9b90 100644 --- a/src/opencc.cc +++ b/src/opencc.cc @@ -17,12 +17,11 @@ #include #include "lib/lua_export_type.h" -#include "lib/luatype_boost_optional.h" +#include "optional.h" using std::string; using std::vector; using std::list; -using boost::optional; using namespace rime; namespace { diff --git a/src/optional.h b/src/optional.h new file mode 100644 index 0000000..e0f6a45 --- /dev/null +++ b/src/optional.h @@ -0,0 +1,7 @@ +#if __cplusplus >= 201703L || _MSVC_LANG >= 201703L +#include "lib/luatype_std_optional.h" +using std::optional; +#else +#include "lib/luatype_boost_optional.h" +using boost::optional; +#endif diff --git a/src/types.cc b/src/types.cc index 8724454..dcaffb3 100644 --- a/src/types.cc +++ b/src/types.cc @@ -20,12 +20,11 @@ #include #include "lib/lua_export_type.h" -#include "lib/luatype_boost_optional.h" +#include "optional.h" #define ENABLE_TYPES_EXT using namespace rime; -using boost::optional; namespace { diff --git a/src/types_ext.cc b/src/types_ext.cc index d66f0e9..66670ec 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -15,12 +15,11 @@ #include #include "lib/lua_export_type.h" -#include "lib/luatype_boost_optional.h" +#include "optional.h" #include using namespace rime; -using boost::optional; namespace { From 403f5d9fe02c5efd58b81b5c2abe217a481a28dc Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Sat, 20 Jan 2024 10:43:32 +0800 Subject: [PATCH 25/47] CommitEntry add update(commit) update(entry, commit, prefix_str) (#289) * CommitEntry add update(commit) update(entry, commit, prefix_str) Signed-off-by: shewer * CommitEntry add update(commit) update(entry, commit, prefix_str) Signed-off-by: shewer --------- Signed-off-by: shewer --- sample/lua/expand_translator.lua | 11 +++++++++- src/types.cc | 36 +++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/sample/lua/expand_translator.lua b/sample/lua/expand_translator.lua index b343952..1aee36b 100644 --- a/sample/lua/expand_translator.lua +++ b/sample/lua/expand_translator.lua @@ -28,11 +28,20 @@ local function memoryCallback(memory, commit) return true end +local function memoryCallback1(commit) + for i, dictentry in ipairs(commit:get()) do + commits:update_entry(dictentry, 1, "") + end + --commits:update(1) -- +end + + local function init(env) env.mem = Memory(env.engine,env.engine.schema) -- ns= "translator" -- env.mem = Memory(env.engine,env.engine.schema, env.name_space ) -- env.mem = Memory(env.engine,Schema("cangjie5") ) -- ns= "translator- - env.mem:memorize(function(commit) memoryCallback(env.mem, commit) end) + --env.mem:memorize(function(commit) memoryCallback(env.mem, commit) end) + env.mem:memorize(memoryCallback1) -- or use -- schema = Schema("cangjie5") -- schema_id -- env.mem = Memory(env.engine, schema, "translator") diff --git a/src/types.cc b/src/types.cc index dcaffb3..9b01022 100644 --- a/src/types.cc +++ b/src/types.cc @@ -1356,10 +1356,33 @@ namespace LogReg { } namespace CommitEntryReg { using T = CommitEntry; + using D = DictEntry; vector get(const T& ce) { return ce.elements; } + bool update_entry(const T &t, const D& entry, int commit, const string& prefix_str) { + if (!t.memory) + return false; + auto user_dict = t.memory->user_dict(); + if (!user_dict || !user_dict->loaded()) + return false; + + return user_dict->UpdateEntry(entry, commit, prefix_str); + } + + bool update(const T& t, int commit) { + if (!t.memory) + return false; + auto user_dict = t.memory->user_dict(); + if (!user_dict || !user_dict->loaded()) + return false; + + for (const DictEntry* e : t.elements) { + user_dict->UpdateEntry(*e, commit); + } + return true; + } static const luaL_Reg funcs[] = { { NULL, NULL }, @@ -1367,6 +1390,8 @@ namespace CommitEntryReg { static const luaL_Reg methods[] = { {"get",WRAP(get)}, + {"update_entry",WRAP(update_entry)}, + {"update",WRAP(update)}, { NULL, NULL }, }; @@ -1380,12 +1405,17 @@ namespace CommitEntryReg { } namespace DictEntryReg { using T = DictEntry; - an make() { - return an(new T()); + + int raw_make(lua_State* L) { + an t = (lua_gettop(L)>0) + ? New(LuaType::todata(L,1)) : New(); + + LuaType>::pushdata(L, t); + return 1; } static const luaL_Reg funcs[] = { - {"DictEntry",WRAP(make)}, + {"DictEntry",raw_make}, { NULL, NULL }, }; From 474b9d95e94e59af3e6d7f7c1b306bce6dae1b62 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Sat, 20 Jan 2024 10:46:21 +0800 Subject: [PATCH 26/47] changed member type on LuaMemory iter, utermember LuaMemory (#290) * changed member type on LuaMemory iter, utermember LuaMemory Signed-off-by: shewer * add DictionaryReg UserDictionaryReg Signed-off-by: shewer --------- Signed-off-by: shewer --- src/types.cc | 313 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 244 insertions(+), 69 deletions(-) diff --git a/src/types.cc b/src/types.cc index 9b01022..387df86 100644 --- a/src/types.cc +++ b/src/types.cc @@ -104,6 +104,8 @@ namespace CandidateReg { using T = Candidate; string dynamic_type(T &c) { + if (dynamic_cast(&c)) + return "Sentence"; if (dynamic_cast(&c)) return "Phrase"; if (dynamic_cast(&c)) @@ -160,6 +162,9 @@ namespace CandidateReg { LOG(WARNING) << "Can\'t append candidate. args #1 expected an " ; return false; }; + an phrase_candidate(an c) { + return std::dynamic_pointer_cast (c); + } static const luaL_Reg funcs[] = { @@ -175,6 +180,7 @@ namespace CandidateReg { { "get_genuines", WRAP(T::GetGenuineCandidates) }, { "to_shadow_candidate", WRAP(shadow_candidate) }, { "to_uniquified_candidate", WRAP(uniquified_candidate) }, + { "to_phrase", WRAP(phrase_candidate) }, { "append", WRAP(append)}, { NULL, NULL }, }; @@ -1481,34 +1487,201 @@ namespace CodeReg { { NULL, NULL }, }; } + +namespace DictionaryReg { + using T = Dictionary; + using I = DictEntryIterator; + using D = DictEntry; + + an lookup_words(T& t, const string& code, bool predictive , size_t limit) { + an ret=New(); + t.LookupWords(ret.get(),code, predictive, limit); + return ret; + } + + vector decode(T& t, const Code& code) { + vector ret; + t.Decode(code, &ret); + return ret; + } + + static const luaL_Reg funcs[] = { + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + { "lookup_words", WRAP(lookup_words)}, + //{ "lookup", WRAPMEM(T, Lookup)}, + { "decode", WRAP(decode)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + { "name", WRAPMEM(T, name)}, + { "loaded", WRAPMEM(T, loaded)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { NULL, NULL }, + }; +} + +namespace UserDictionaryReg { + using T = UserDictionary; + using I = UserDictEntryIterator; + using D = DictEntry; + + an lookup_words(T& t, const string& code, bool predictive , size_t limit) { + an ret=New(); + t.LookupWords(ret.get(),code, predictive, limit); + return ret; + } + bool update_entry(T& t, const D& entry, int commits, const string& prefix) { + return t.UpdateEntry(entry, commits, prefix); + } + + static const luaL_Reg funcs[] = { + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + { "lookup_words", WRAP(lookup_words)}, + //{ "lookup", WRAPMEM(T, Lookup)}, + { "update_entry", WRAP(update_entry)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + { "name", WRAPMEM(T, name)}, + { "loaded", WRAPMEM(T, loaded)}, + { "tick", WRAPMEM(T, tick)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { NULL, NULL }, + }; +} + +namespace DictEntryIteratorReg { + using T = DictEntryIterator; + using D = DictEntry; + + optional> Next(T& t) { + if ( t.exhausted()) { + return {}; + } + an ret = t.Peek(); + t.Next(); + return ret; + } + + int raw_iter(lua_State* L) { + int n = lua_gettop(L); + if (n>=1) { // :iter() + lua_pushcfunction(L, WRAP(Next)); + lua_insert(L, 1); + return 2; + } + return 0; + } + + static const luaL_Reg funcs[] = { + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + {"iter", raw_iter}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { NULL, NULL }, + }; + +} + +namespace UserDictEntryIteratorReg { + using T = UserDictEntryIterator; + using D = DictEntry; + + optional> Next(T& t) { + if ( t.exhausted()) { + return {}; + } + an ret = t.Peek(); + t.Next(); + return ret; + } + + int raw_iter(lua_State* L) { + int n = lua_gettop(L); + if (n>=1) { // :iter() + lua_pushcfunction(L, WRAP(Next)); + lua_insert(L, 1); + return 2; + } + return 0; + } + + static const luaL_Reg funcs[] = { + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + {"iter", raw_iter}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { NULL, NULL }, + }; +} namespace MemoryReg { class LuaMemory : public Memory { an memorize_callback; Lua *lua_; public: - using Memory::Memory; - DictEntryIterator iter; - UserDictEntryIterator uter; - - LuaMemory(Lua *lua, const Ticket& ticket) - : lua_(lua), Memory(ticket) {} + an iter; + an uter; + an dictLookup(const string& str_code, bool predictive, size_t expand_search_limit); + an userLookup(const string& input, const bool isExpand); + vector decode(const Code& code); + bool update_userdict(const DictEntry& entry, const int commits, const string& new_entry_prefix); virtual bool Memorize(const CommitEntry&); + LuaMemory(const Ticket& ticket, Lua* lua) + : lua_(lua), Memory(ticket) {} + void memorize(an func) { memorize_callback = func; } - void clearDict() { - iter = DictEntryIterator(); + iter.reset(); } void clearUser() { - uter = UserDictEntryIterator(); + uter.reset(); } }; - using T = LuaMemory; - bool MemoryReg::LuaMemory::Memorize(const CommitEntry& commit_entry) { + vector LuaMemory::decode(const Code& code) { + vector res; + if (dict_ && dict_->loaded()) + dict_->Decode(code,&res); + return res; + } + + bool LuaMemory::Memorize(const CommitEntry& commit_entry) { if (!memorize_callback) return false; @@ -1521,8 +1694,36 @@ namespace MemoryReg { return r.get(); } + an LuaMemory::dictLookup(const string& input, const bool isExpand,size_t limit) { + iter = New();// t= New(); + limit = limit == 0 ? 0xffffffffffffffff : limit; + if (dict_ && dict_->loaded()) { + dict_->LookupWords(iter.get(), input, isExpand, limit); + } + return iter; + } + + an LuaMemory::userLookup(const string& input, const bool isExpand) { + uter = New(); + if (user_dict_ && user_dict_->loaded()) { + user_dict_->LookupWords(uter.get(), input, isExpand); + } + return uter; + } + + bool LuaMemory::update_userdict(const DictEntry& entry, const int commits, const string& new_entry_prefix) { + if (user_dict_ && user_dict_->loaded()) + return user_dict_->UpdateEntry(entry, commits, new_entry_prefix); + + return false; + } + + // XXX: Currently the WRAP macro is not generic enough, // so that we need a raw function to get the lua state / parse variable args. + + using T = LuaMemory; + int raw_make(lua_State *L) { // TODO: fix the memory leak C_State C; @@ -1538,63 +1739,36 @@ namespace MemoryReg { if (3 <= n) translatorTicket.name_space = LuaType::todata(L, 3, &C); - an memoli = New(lua, translatorTicket); + an memoli = New(translatorTicket, lua); LuaType>::pushdata(L, memoli); return 1; } - bool dictLookup(T& memory, const string& input, const bool isExpand,size_t limit) { - memory.clearDict(); - limit = limit == 0 ? 0xffffffffffffffff : limit; - if (auto dict = memory.dict()) - return dict->LookupWords(&memory.iter, input, isExpand, limit) > 0; - else - return false; - } - - optional> dictNext(T& memory) { - if (memory.iter.exhausted()) { - return {}; - } - an ret = memory.iter.Peek(); - memory.iter.Next(); - return ret; - } - - bool userLookup(T& memory, const string& input, const bool isExpand) { - memory.clearUser(); - if (auto dict = memory.user_dict()) - return dict->LookupWords(&memory.uter, input, isExpand) > 0; - else - return false; - } - - optional> userNext(T& memory) { - if (memory.uter.exhausted()) { - return {}; - } - an ret = memory.uter.Peek(); - memory.uter.Next(); - return ret; - } - - bool updateToUserdict(T& memory, const DictEntry& entry, const int commits, const string& new_entry_prefix) { - if (auto dict = memory.user_dict()) - return dict->UpdateEntry(entry, commits, new_entry_prefix); - else - return false; - } + // :iter_user([func[,...]]) return next_func, entryiterator + // return next_entry, enter_iter[,func[, ...]] int raw_iter_user(lua_State* L) { - lua_pushcfunction(L, WRAP(userNext)); - lua_pushvalue(L, 1); - return 2; + an t = LuaType>::todata(L, 1); + LuaType>::pushdata(L, t->uter); + lua_replace(L, 1); + lua_getfield(L, 1, "iter"); + lua_insert(L, 1); + if (lua_pcall(L, lua_gettop(L) -1 , 2,0) != LUA_OK) + return 0; + return lua_gettop(L); } + // :iter_user([func[,...]]) return next_func, entryiterator + // return next_entry, enter_iter[,func[, ...]] int raw_iter_dict(lua_State* L) { - lua_pushcfunction(L, WRAP(dictNext)); - lua_pushvalue(L, 1); - return 2; + an t = LuaType>::todata(L, 1); + LuaType>::pushdata(L, t->iter); + lua_replace(L, 1); + lua_getfield(L, 1, "iter"); + lua_insert(L, 1); + if (lua_pcall(L, lua_gettop(L) -1 , 2,0) != LUA_OK) + return 0; + return lua_gettop(L); } static const luaL_Reg funcs[] = { @@ -1602,24 +1776,21 @@ namespace MemoryReg { {NULL, NULL}, }; - std::vector decode(T& memory, Code& code) { - std::vector res; - if (auto dict = memory.dict()) - dict->Decode(code,&res); - return res; - } static const luaL_Reg methods[] = { - { "dict_lookup", WRAP(dictLookup)}, - { "user_lookup", WRAP(userLookup)}, + { "dict_lookup", WRAPMEM(T::dictLookup)}, + { "user_lookup", WRAPMEM(T::userLookup)}, { "memorize", WRAPMEM(T::memorize)}, - { "decode", WRAP(decode)}, + { "decode", WRAPMEM(T::decode)}, { "iter_dict", raw_iter_dict}, { "iter_user", raw_iter_user}, - { "update_userdict", WRAP(updateToUserdict)}, + { "update_userdict", WRAPMEM(T::update_userdict)}, + { "update_entry", WRAPMEM(T::update_userdict)}, {NULL, NULL}, }; static const luaL_Reg vars_get[] = { + { "dict", WRAPMEM(T, dict)}, + { "user_dict", WRAPMEM(T, user_dict)}, {NULL, NULL}, }; @@ -1878,6 +2049,10 @@ void types_init(lua_State *L) { EXPORT(KeyEventNotifierReg, L); EXPORT(ConnectionReg, L); EXPORT(MemoryReg, L); + EXPORT(DictionaryReg, L); + EXPORT(UserDictionaryReg, L); + EXPORT(DictEntryIteratorReg, L); + EXPORT(UserDictEntryIteratorReg, L); EXPORT(DictEntryReg, L); EXPORT(CodeReg, L); EXPORT(CommitEntryReg, L); From db435b8a4c14d4abc6a5b2a57119c049e5b16900 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Sat, 20 Jan 2024 10:56:47 +0800 Subject: [PATCH 27/47] fix build with external lua (#300) --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ff75973..356731e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,16 +2,17 @@ if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/lua5.4/lua.h") find_package(PkgConfig) if(PkgConfig_FOUND) foreach(pkg lua lua54 lua53 lua52 luajit lua51) - pkg_check_modules(LUA ${pkg}) + pkg_check_modules(LUA IMPORTED_TARGET GLOBAL ${pkg}) if(LUA_FOUND) break() endif() endforeach() endif() if(LUA_FOUND) + set(LUA_TARGET PkgConfig::LUA) include_directories(${LUA_INCLUDE_DIRS}) else() - message(FATAL_ERROR "Lua not found, consider using `bash travis-install.sh` to download.") + message(FATAL_ERROR "Lua not found, consider using `bash action-install.sh` to download.") endif() else() message(STATUS "Using in-tree lua source") @@ -44,5 +45,5 @@ endif() set(plugin_name "rime-lua" PARENT_SCOPE) set(plugin_objs $ PARENT_SCOPE) -set(plugin_deps ${LUA_LIBRARIES} ${rime_library} ${rime_gears_library} PARENT_SCOPE) +set(plugin_deps ${LUA_TARGET} ${rime_library} ${rime_gears_library} PARENT_SCOPE) set(plugin_modules "lua" PARENT_SCOPE) From f7e10fda9a507ff331524ab7f656a1e8b46eeb14 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Sat, 20 Jan 2024 10:59:13 +0800 Subject: [PATCH 28/47] add SentenceReg (#291) Signed-off-by: shewer Co-authored-by: hchunhui --- src/types.cc | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/src/types.cc b/src/types.cc index 387df86..e2c09f1 100644 --- a/src/types.cc +++ b/src/types.cc @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -166,6 +167,10 @@ namespace CandidateReg { return std::dynamic_pointer_cast (c); } + template + an candidate_to_(an t) { + return std::dynamic_pointer_cast(t); + }; static const luaL_Reg funcs[] = { { "Candidate", WRAP(make) }, @@ -180,7 +185,8 @@ namespace CandidateReg { { "get_genuines", WRAP(T::GetGenuineCandidates) }, { "to_shadow_candidate", WRAP(shadow_candidate) }, { "to_uniquified_candidate", WRAP(uniquified_candidate) }, - { "to_phrase", WRAP(phrase_candidate) }, + { "to_phrase", WRAP(candidate_to_)}, + { "to_sentence", WRAP(candidate_to_)}, { "append", WRAP(append)}, { NULL, NULL }, }; @@ -1816,6 +1822,10 @@ namespace PhraseReg { return phrase; } + string lang_name(T &t){ + return t.language()->name(); + } + static const luaL_Reg funcs[] = { { "Phrase", WRAP(make) }, { NULL, NULL }, @@ -1828,6 +1838,7 @@ namespace PhraseReg { static const luaL_Reg vars_get[] = { { "language", WRAPMEM(T, language)}, + { "lang_name", WRAP(lang_name)}, { "type", WRAPMEM(T, type) }, { "start", WRAPMEM(T, start) }, { "_start", WRAPMEM(T, start) }, @@ -1857,6 +1868,72 @@ namespace PhraseReg { { NULL, NULL }, }; }// Phrase work with Translator +//--- wrappers for Phrase +namespace SentenceReg { + using T = Sentence; + + an toCandidate(an t) { + return t; + } + + string lang_name(T& t){ + return t.language()->name(); + } + + vector components(T& t) { + return t.components(); + } + + vector word_lengths(T& t) { + return t.word_lengths(); + } + + static const luaL_Reg funcs[] = { + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + { "toCandidate", WRAP(toCandidate)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + { "language", WRAPMEM(T, language)}, + { "lang_name", WRAP(lang_name)}, + { "type", WRAPMEM(T, type) }, + { "start", WRAPMEM(T, start) }, + { "_start", WRAPMEM(T, start) }, + { "_end", WRAPMEM(T, end) }, // end is keyword in Lua... + { "quality", WRAPMEM(T, quality) }, + { "text", WRAPMEM(T, text) }, + { "comment", WRAPMEM(T, comment) }, + { "preedit", WRAPMEM(T, preedit) }, + { "weight", WRAPMEM(T, weight)}, + { "code", WRAPMEM(T, code)}, + { "entry", WRAPMEM(T, entry)}, + //span + //language doesn't wrap yet, so Wrap it later + // Sentence membors + { "word_lengths", WRAP(word_lengths)}, + { "entrys", WRAP(components)}, + { "entrys_size", WRAPMEM(T, size)}, + { "entrys_empty", WRAPMEM(T, empty)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { "type", WRAPMEM(T, set_type) }, + { "start", WRAPMEM(T, set_start) }, + { "_start", WRAPMEM(T, set_start) }, + { "_end", WRAPMEM(T, set_end) }, + { "quality", WRAPMEM(T, set_quality) }, + { "comment", WRAPMEM(T, set_comment) }, + { "preedit", WRAPMEM(T, set_preedit) }, + { "weight", WRAPMEM(T, set_weight)}, + // set_syllabifier + { NULL, NULL }, + }; +}// Sentence work with Translator namespace KeySequenceReg { using T = KeySequence; @@ -2057,6 +2134,7 @@ void types_init(lua_State *L) { EXPORT(CodeReg, L); EXPORT(CommitEntryReg, L); EXPORT(PhraseReg, L); + EXPORT(SentenceReg, L); EXPORT(KeySequenceReg, L); EXPORT(SwitcherReg, L); LogReg::init(L); From 9c611a15277b9978960df780a8f5ca3dee630354 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Sat, 20 Jan 2024 11:11:12 +0800 Subject: [PATCH 29/47] =?UTF-8?q?=20=E5=A2=9E=E5=8A=A0=20ShadowCandidate()?= =?UTF-8?q?=20,UniquifiedCandidate()=20=E4=B8=8D=E5=AE=9A=E5=8F=83?= =?UTF-8?q?=E6=95=B8=20(#296)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原 ShadowCandidate( cand, type, text, comment) -- args 4 改 ShadowCandidate( cand, type [ ,text [,comment [,inherit_comment]]]) -- args 2~5 原 UniquifiedCandidate( cand, type, text, comment) -- args 4 改 UniquifiedCandidate( cand, type [, text [, comment]]) -- args 2~4 Co-authored-by: hchunhui --- src/types.cc | 52 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/src/types.cc b/src/types.cc index e2c09f1..22b36c2 100644 --- a/src/types.cc +++ b/src/types.cc @@ -145,16 +145,58 @@ namespace CandidateReg { } an shadow_candidate(const an item, - const string& type, const string& text, const string& comment) + const string& type, const string& text, const string& comment, + const bool inherit_comment) { return New(item, type, text, comment); } + int raw_shadow_candidate(lua_State* L) { + size_t n = lua_gettop(L); + if (2 > n) { + return (1 == n) ? + luaL_error(L, "bad argument #2 to func (string expected, got no value)") : + luaL_error(L, "bad argument #1 to func (an expected, got no value)"); + } + // init args(2-5) ( an, type [,text, comment, inherit_comment]) + if (5 < n) + lua_pop(L, n-5); + else if (4 == n) + lua_pushboolean(L, true); + else if (4 > n) { + for (int i=n; 4 > i ; i++) + lua_pushstring(L, ""); + lua_pushboolean(L, true); + } + lua_pushcfunction(L, WRAP(shadow_candidate)); + lua_insert(L, 1); + return (LUA_OK==lua_pcall(L, lua_gettop(L)-1, 1, 0)) ? 1 : 0; + } + an uniquified_candidate(const an item, const string& type, const string& text, const string& comment) { return New(item, type, text, comment); } + int raw_uniquified_candidate(lua_State* L) { + size_t n = lua_gettop(L); + if (2 > n) { + return (1 == n) ? + luaL_error(L, "bad argument #2 to func (string expected, got no value)") : + luaL_error(L, "bad argument #1 to func (an expected, got no value)"); + } + // init args(2-4) ( an, type [,text, comment]) + if (4 < n) + lua_pop(L, n-4); + else if (4 > n) { + for (int i=n; 4 > i ; i++) + lua_pushstring(L, ""); + } + lua_pushcfunction(L, WRAP(uniquified_candidate)); + lua_insert(L, 1); + return (LUA_OK==lua_pcall(L, lua_gettop(L)-1, 1, 0)) ? 1 : 0; + } + bool append(an self, an item) { if (auto cand= As(self) ) { cand->Append(item); @@ -174,8 +216,8 @@ namespace CandidateReg { static const luaL_Reg funcs[] = { { "Candidate", WRAP(make) }, - { "ShadowCandidate", WRAP(shadow_candidate) }, - { "UniquifiedCandidate", WRAP(uniquified_candidate) }, + { "ShadowCandidate", (raw_shadow_candidate) }, + { "UniquifiedCandidate", (raw_uniquified_candidate) }, { NULL, NULL }, }; @@ -183,8 +225,8 @@ namespace CandidateReg { { "get_dynamic_type", WRAP(dynamic_type) }, { "get_genuine", WRAP(T::GetGenuineCandidate) }, { "get_genuines", WRAP(T::GetGenuineCandidates) }, - { "to_shadow_candidate", WRAP(shadow_candidate) }, - { "to_uniquified_candidate", WRAP(uniquified_candidate) }, + { "to_shadow_candidate", (raw_shadow_candidate) }, + { "to_uniquified_candidate", (raw_uniquified_candidate) }, { "to_phrase", WRAP(candidate_to_)}, { "to_sentence", WRAP(candidate_to_)}, { "append", WRAP(append)}, From 265e1cca0ecb52a26f95e4a539b51714721617a4 Mon Sep 17 00:00:00 2001 From: Chunhui He Date: Tue, 30 Jan 2024 11:07:20 +0000 Subject: [PATCH 30/47] Revert "changed member type on LuaMemory iter, utermember LuaMemory (#290)" This reverts commit 474b9d95e94e59af3e6d7f7c1b306bce6dae1b62. --- src/types.cc | 312 ++++++++++++--------------------------------------- 1 file changed, 69 insertions(+), 243 deletions(-) diff --git a/src/types.cc b/src/types.cc index 22b36c2..21fdd9f 100644 --- a/src/types.cc +++ b/src/types.cc @@ -105,8 +105,6 @@ namespace CandidateReg { using T = Candidate; string dynamic_type(T &c) { - if (dynamic_cast(&c)) - return "Sentence"; if (dynamic_cast(&c)) return "Phrase"; if (dynamic_cast(&c)) @@ -205,9 +203,6 @@ namespace CandidateReg { LOG(WARNING) << "Can\'t append candidate. args #1 expected an " ; return false; }; - an phrase_candidate(an c) { - return std::dynamic_pointer_cast (c); - } template an candidate_to_(an t) { @@ -1535,201 +1530,34 @@ namespace CodeReg { { NULL, NULL }, }; } - -namespace DictionaryReg { - using T = Dictionary; - using I = DictEntryIterator; - using D = DictEntry; - - an lookup_words(T& t, const string& code, bool predictive , size_t limit) { - an ret=New(); - t.LookupWords(ret.get(),code, predictive, limit); - return ret; - } - - vector decode(T& t, const Code& code) { - vector ret; - t.Decode(code, &ret); - return ret; - } - - static const luaL_Reg funcs[] = { - { NULL, NULL }, - }; - - static const luaL_Reg methods[] = { - { "lookup_words", WRAP(lookup_words)}, - //{ "lookup", WRAPMEM(T, Lookup)}, - { "decode", WRAP(decode)}, - { NULL, NULL }, - }; - - static const luaL_Reg vars_get[] = { - { "name", WRAPMEM(T, name)}, - { "loaded", WRAPMEM(T, loaded)}, - { NULL, NULL }, - }; - - static const luaL_Reg vars_set[] = { - { NULL, NULL }, - }; -} - -namespace UserDictionaryReg { - using T = UserDictionary; - using I = UserDictEntryIterator; - using D = DictEntry; - - an lookup_words(T& t, const string& code, bool predictive , size_t limit) { - an ret=New(); - t.LookupWords(ret.get(),code, predictive, limit); - return ret; - } - bool update_entry(T& t, const D& entry, int commits, const string& prefix) { - return t.UpdateEntry(entry, commits, prefix); - } - - static const luaL_Reg funcs[] = { - { NULL, NULL }, - }; - - static const luaL_Reg methods[] = { - { "lookup_words", WRAP(lookup_words)}, - //{ "lookup", WRAPMEM(T, Lookup)}, - { "update_entry", WRAP(update_entry)}, - { NULL, NULL }, - }; - - static const luaL_Reg vars_get[] = { - { "name", WRAPMEM(T, name)}, - { "loaded", WRAPMEM(T, loaded)}, - { "tick", WRAPMEM(T, tick)}, - { NULL, NULL }, - }; - - static const luaL_Reg vars_set[] = { - { NULL, NULL }, - }; -} - -namespace DictEntryIteratorReg { - using T = DictEntryIterator; - using D = DictEntry; - - optional> Next(T& t) { - if ( t.exhausted()) { - return {}; - } - an ret = t.Peek(); - t.Next(); - return ret; - } - - int raw_iter(lua_State* L) { - int n = lua_gettop(L); - if (n>=1) { // :iter() - lua_pushcfunction(L, WRAP(Next)); - lua_insert(L, 1); - return 2; - } - return 0; - } - - static const luaL_Reg funcs[] = { - { NULL, NULL }, - }; - - static const luaL_Reg methods[] = { - {"iter", raw_iter}, - { NULL, NULL }, - }; - - static const luaL_Reg vars_get[] = { - { NULL, NULL }, - }; - - static const luaL_Reg vars_set[] = { - { NULL, NULL }, - }; - -} - -namespace UserDictEntryIteratorReg { - using T = UserDictEntryIterator; - using D = DictEntry; - - optional> Next(T& t) { - if ( t.exhausted()) { - return {}; - } - an ret = t.Peek(); - t.Next(); - return ret; - } - - int raw_iter(lua_State* L) { - int n = lua_gettop(L); - if (n>=1) { // :iter() - lua_pushcfunction(L, WRAP(Next)); - lua_insert(L, 1); - return 2; - } - return 0; - } - - static const luaL_Reg funcs[] = { - { NULL, NULL }, - }; - - static const luaL_Reg methods[] = { - {"iter", raw_iter}, - { NULL, NULL }, - }; - - static const luaL_Reg vars_get[] = { - { NULL, NULL }, - }; - - static const luaL_Reg vars_set[] = { - { NULL, NULL }, - }; -} namespace MemoryReg { class LuaMemory : public Memory { an memorize_callback; Lua *lua_; public: - an iter; - an uter; - - an dictLookup(const string& str_code, bool predictive, size_t expand_search_limit); - an userLookup(const string& input, const bool isExpand); - vector decode(const Code& code); - bool update_userdict(const DictEntry& entry, const int commits, const string& new_entry_prefix); - virtual bool Memorize(const CommitEntry&); + using Memory::Memory; + DictEntryIterator iter; + UserDictEntryIterator uter; - LuaMemory(const Ticket& ticket, Lua* lua) + LuaMemory(Lua *lua, const Ticket& ticket) : lua_(lua), Memory(ticket) {} + virtual bool Memorize(const CommitEntry&); + void memorize(an func) { memorize_callback = func; } + void clearDict() { - iter.reset(); + iter = DictEntryIterator(); } void clearUser() { - uter.reset(); + uter = UserDictEntryIterator(); } }; + using T = LuaMemory; - vector LuaMemory::decode(const Code& code) { - vector res; - if (dict_ && dict_->loaded()) - dict_->Decode(code,&res); - return res; - } - - bool LuaMemory::Memorize(const CommitEntry& commit_entry) { + bool MemoryReg::LuaMemory::Memorize(const CommitEntry& commit_entry) { if (!memorize_callback) return false; @@ -1742,36 +1570,8 @@ namespace MemoryReg { return r.get(); } - an LuaMemory::dictLookup(const string& input, const bool isExpand,size_t limit) { - iter = New();// t= New(); - limit = limit == 0 ? 0xffffffffffffffff : limit; - if (dict_ && dict_->loaded()) { - dict_->LookupWords(iter.get(), input, isExpand, limit); - } - return iter; - } - - an LuaMemory::userLookup(const string& input, const bool isExpand) { - uter = New(); - if (user_dict_ && user_dict_->loaded()) { - user_dict_->LookupWords(uter.get(), input, isExpand); - } - return uter; - } - - bool LuaMemory::update_userdict(const DictEntry& entry, const int commits, const string& new_entry_prefix) { - if (user_dict_ && user_dict_->loaded()) - return user_dict_->UpdateEntry(entry, commits, new_entry_prefix); - - return false; - } - - // XXX: Currently the WRAP macro is not generic enough, // so that we need a raw function to get the lua state / parse variable args. - - using T = LuaMemory; - int raw_make(lua_State *L) { // TODO: fix the memory leak C_State C; @@ -1787,36 +1587,63 @@ namespace MemoryReg { if (3 <= n) translatorTicket.name_space = LuaType::todata(L, 3, &C); - an memoli = New(translatorTicket, lua); + an memoli = New(lua, translatorTicket); LuaType>::pushdata(L, memoli); return 1; } + bool dictLookup(T& memory, const string& input, const bool isExpand,size_t limit) { + memory.clearDict(); + limit = limit == 0 ? 0xffffffffffffffff : limit; + if (auto dict = memory.dict()) + return dict->LookupWords(&memory.iter, input, isExpand, limit) > 0; + else + return false; + } + + optional> dictNext(T& memory) { + if (memory.iter.exhausted()) { + return {}; + } + an ret = memory.iter.Peek(); + memory.iter.Next(); + return ret; + } + + bool userLookup(T& memory, const string& input, const bool isExpand) { + memory.clearUser(); + if (auto dict = memory.user_dict()) + return dict->LookupWords(&memory.uter, input, isExpand) > 0; + else + return false; + } + + optional> userNext(T& memory) { + if (memory.uter.exhausted()) { + return {}; + } + an ret = memory.uter.Peek(); + memory.uter.Next(); + return ret; + } + + bool updateToUserdict(T& memory, const DictEntry& entry, const int commits, const string& new_entry_prefix) { + if (auto dict = memory.user_dict()) + return dict->UpdateEntry(entry, commits, new_entry_prefix); + else + return false; + } - // :iter_user([func[,...]]) return next_func, entryiterator - // return next_entry, enter_iter[,func[, ...]] int raw_iter_user(lua_State* L) { - an t = LuaType>::todata(L, 1); - LuaType>::pushdata(L, t->uter); - lua_replace(L, 1); - lua_getfield(L, 1, "iter"); - lua_insert(L, 1); - if (lua_pcall(L, lua_gettop(L) -1 , 2,0) != LUA_OK) - return 0; - return lua_gettop(L); + lua_pushcfunction(L, WRAP(userNext)); + lua_pushvalue(L, 1); + return 2; } - // :iter_user([func[,...]]) return next_func, entryiterator - // return next_entry, enter_iter[,func[, ...]] int raw_iter_dict(lua_State* L) { - an t = LuaType>::todata(L, 1); - LuaType>::pushdata(L, t->iter); - lua_replace(L, 1); - lua_getfield(L, 1, "iter"); - lua_insert(L, 1); - if (lua_pcall(L, lua_gettop(L) -1 , 2,0) != LUA_OK) - return 0; - return lua_gettop(L); + lua_pushcfunction(L, WRAP(dictNext)); + lua_pushvalue(L, 1); + return 2; } static const luaL_Reg funcs[] = { @@ -1824,21 +1651,24 @@ namespace MemoryReg { {NULL, NULL}, }; + std::vector decode(T& memory, Code& code) { + std::vector res; + if (auto dict = memory.dict()) + dict->Decode(code,&res); + return res; + } static const luaL_Reg methods[] = { - { "dict_lookup", WRAPMEM(T::dictLookup)}, - { "user_lookup", WRAPMEM(T::userLookup)}, + { "dict_lookup", WRAP(dictLookup)}, + { "user_lookup", WRAP(userLookup)}, { "memorize", WRAPMEM(T::memorize)}, - { "decode", WRAPMEM(T::decode)}, + { "decode", WRAP(decode)}, { "iter_dict", raw_iter_dict}, { "iter_user", raw_iter_user}, - { "update_userdict", WRAPMEM(T::update_userdict)}, - { "update_entry", WRAPMEM(T::update_userdict)}, + { "update_userdict", WRAP(updateToUserdict)}, {NULL, NULL}, }; static const luaL_Reg vars_get[] = { - { "dict", WRAPMEM(T, dict)}, - { "user_dict", WRAPMEM(T, user_dict)}, {NULL, NULL}, }; @@ -2168,10 +1998,6 @@ void types_init(lua_State *L) { EXPORT(KeyEventNotifierReg, L); EXPORT(ConnectionReg, L); EXPORT(MemoryReg, L); - EXPORT(DictionaryReg, L); - EXPORT(UserDictionaryReg, L); - EXPORT(DictEntryIteratorReg, L); - EXPORT(UserDictEntryIteratorReg, L); EXPORT(DictEntryReg, L); EXPORT(CodeReg, L); EXPORT(CommitEntryReg, L); From a29897dc3dd47b2849d5ed59c81261449842bd53 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Fri, 2 Feb 2024 20:04:56 +0800 Subject: [PATCH 31/47] Add TableTranslator , ScriptTranslator in Component (#287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add Component.TableTranslator Signed-off-by: shewer * add LTableTranslator * 移除 table_translator.h Signed-off-by: shewer * 移除 table_translator.h Signed-off-by: shewer * 加入 LScriptTranslator Signed-off-by: shewer * fixed boost::optional error Signed-off-by: shewer * fix memory dump : memories_callback 初始化 Signed-off-by: shewer * add sample Signed-off-by: shewer * fixd code Signed-off-by: shewer * clang-format 排版 Signed-off-by: shewer * fixed sample/ script code * 補上 *engine Signed-off-by: shewer * add dict user_dict members of Translator Signed-off-by: shewer * refactoring code Signed-off-by: shewer --------- Signed-off-by: shewer --- sample/lua/script_translator.lua | 81 +++++++++++ sample/lua/table_translator.lua | 84 ++++++++++++ sample/luna_pinyin.custom.yaml | 6 + src/script_translator.cc | 178 ++++++++++++++++++++++++ src/table_translator.cc | 223 +++++++++++++++++++++++++++++++ src/translator.h | 69 ++++++++++ src/types_ext.cc | 17 ++- 7 files changed, 652 insertions(+), 6 deletions(-) create mode 100644 sample/lua/script_translator.lua create mode 100644 sample/lua/table_translator.lua create mode 100644 sample/luna_pinyin.custom.yaml create mode 100644 src/script_translator.cc create mode 100644 src/table_translator.cc create mode 100644 src/translator.h diff --git a/sample/lua/script_translator.lua b/sample/lua/script_translator.lua new file mode 100644 index 0000000..fac3955 --- /dev/null +++ b/sample/lua/script_translator.lua @@ -0,0 +1,81 @@ +#! /usr/bin/env lua +-- +-- script_translator.lua +-- Copyright (C) 2023 Shewer Lu +-- +-- Distributed under terms of the MIT license. +-- + +--[[ + +''' custom.yaml' +patch: + engine/translators/+: + - lua_translator@*table_translator@translator + - lua_translator@*script_translator@cangjie +```` +------- methods return +env.tran:start_session false function() +env.tran:finish_session false function() +env.tran:discard_session false function() +env.tran:query false function(inp, seg) +env.tran:memorize false function(commit_entrys) +env.tran:update_entry false function(entry, state, prefix_str) +env.tran.memorize_callback = function(self, commit_entry) +------- vars_set +env.tran.spelling_hints = int >0 +env.tran.initial_quality = double +env.tran.contextual_suggestions = boolean +env.tran.enable_completion = boolean +env.tran.always_show_comments = boolean +env.tran.strict_spelling = boolean +env.tran.max_homophones = int +env.tran.enable_correction = boolean +env.tran.tag = string +env.tran.delimiters = string + +------- vars_get +res = env.tran.spelling_hints 0 number +res = env.tran.initial_quality 0.0 number +res = env.tran.contextual_suggestions false boolean +res = env.tran.enable_completion true boolean +res = env.tran.always_show_comments false boolean +res = env.tran.strict_spelling false boolean +res = env.tran.max_homophones 1 number +res = env.tran.enable_correction false boolean +res = env.tran.tag abc string +res = env.tran.delimiters ' string + +--]] +local M={} +local function simple_callback(self, commits) + local context = self.engine.context + if true then + return self:memorize(commits) + end +end +local function callback(self, commits) -- self : env.tran commits : list + local context = self.engine.context + for i, entry in ipairs(commits:get()) do + self:update_entry(entry,0,"") -- do nothing to userdict + -- self:update_entry(entry,1,"") -- update entry to userdict + -- self:update_entry(entry,-1,"") -- delete entry to userdict + end +end +function M.init(env) + env.tran = Component.ScriptTranslator(env.engine, env.name_space, "script_translator") + env.tran:memorize_callback(simple_callback) +end + +function M.fini(env) +end + +function M.func(inp, seg, env) + local t = env.tran:query(inp,seg) + if not t then return end + for cand in t:iter() do + yield(cand) + end +end + +return M diff --git a/sample/lua/table_translator.lua b/sample/lua/table_translator.lua new file mode 100644 index 0000000..aa92681 --- /dev/null +++ b/sample/lua/table_translator.lua @@ -0,0 +1,84 @@ +#! /usr/bin/env lua +-- +-- table_translator.lua +-- Copyright (C) 2023 Shewer Lu +-- +-- Distributed under terms of the MIT license. +-- +--[[ +-- +''' custom.yaml' +patch: + engine/translators/+: + - lua_translator@*table_translator@translator + - lua_translator@*script_translator@cangjie +```` + +------- methods +env.tran:finish_session bool function() +env.tran:start_session bool function() +env.tran:discard_session bool function() +env.tran:query translation function(inp, seg) +env.tran:memorize bool function(commit_entrys) +env.tran:update_entry bool function(entry) + +------- vars_set 設定值 +env.tran.max_homophones = number +env.tran.spelling_hints = number +env.tran.enable_correction = boolean +env.tran.memorize_callback = function(self, commits) +env.tran.enable_completion = false boolean +env.tran.delimiters = false string +env.tran.strict_spelling = boolean +env.tran.contextual_suggestions = boolean +env.tran.initial_quality = double +env.tran.always_show_comments = boolean +env.tran.tag = string + +------- vars_get 取值 +res = env.tran.max_homophones 1 number +res = env.tran.spelling_hints 0 number +res = env.tran.enable_correction false boolean +res = env.tran.enable_completion true boolean +res = env.tran.delimiters ' string +res = env.tran.strict_spelling false boolean +res = env.tran.contextual_suggestions false boolean +res = env.tran.initial_quality 0.0 number +res = env.tran.always_show_comments false boolean +res = env.tran.tag cangjie string + + +--]] + +local M={} +local function simple_callback(self, commits) + local context = self.engine.context + if true then + return self:memorize(commits) + end +end +local function callback(self, commits) -- self : env.tran commits : list + local context = self.engine.context + for i, entry in ipairs(commits:get()) do + self:update_entry(entry, 0,"") -- do nothing to userdict + -- self:update_entry(entry,1,"") -- update entry to userdict + -- self:update_entry(entry,-1,"") -- delete entry to userdict + end +end +function M.init(env) + env.tran = Component.TableTranslator(env.engine, env.name_space, "table_translator") + env.tran:memorize_callback(simple_callback) +end + +function M.fini(env) +end + +function M.func(inp, seg, env) + local t = env.tran:query(inp,seg) + if not t then return end + for cand in t:iter() do + yield(cand) + end +end + +return M diff --git a/sample/luna_pinyin.custom.yaml b/sample/luna_pinyin.custom.yaml new file mode 100644 index 0000000..dc85bd3 --- /dev/null +++ b/sample/luna_pinyin.custom.yaml @@ -0,0 +1,6 @@ +patch: + engine/translators/+: + - lua_translator@*table_translator@translator + - lua_translator@*script_translator@cangjie + + diff --git a/src/script_translator.cc b/src/script_translator.cc new file mode 100644 index 0000000..e604955 --- /dev/null +++ b/src/script_translator.cc @@ -0,0 +1,178 @@ +/* + * table_translator.cc + * Copyright (C) 2023 Shewer Lu + * + * Distributed under terms of the MIT license. + */ + +#include +#include +#include +#include + + +#include "translator.h" + +using namespace rime; + +namespace { +namespace ScriptTranslatorReg { + + class LScriptTranslator : public ScriptTranslator { + public: + LScriptTranslator(const Ticket& ticket, Lua* lua) + : ScriptTranslator(ticket), lua_(lua) {}; + + virtual bool Memorize(const CommitEntry& commit_entry); + bool memorize(const CommitEntry& commit_entry); + bool update_entry(const DictEntry& index, + int commits, const string& new_entory_prefix); + + SET_(memorize_callback, an); + bool memorize_callback(); + + // TranslatorOptions + SET_(contextual_suggestions, bool); + SET_(delimiters, string&); + SET_(preedit_formatter, Projection&); + SET_(comment_formatter, Projection&); + bool reload_user_dict_disabling_patterns(an); + + // ScriptTranslator member + ACCESS_(spelling_hints, int); + ACCESS_(always_show_comments, bool); + ACCESS_(max_homophones, int); + GET_(enable_correction, bool); + void set_enable_correction(bool); + + protected: + Lua* lua_; + an memorize_callback_; + void init_correction(); + }; + + using T = LScriptTranslator; + + bool T::memorize_callback() { + return (memorize_callback_) ? true : false; + } + + bool T::memorize(const CommitEntry& commit_entry) { + return ScriptTranslator::Memorize(commit_entry); + } + + bool T::Memorize(const CommitEntry& commit_entry) { + if (!memorize_callback_) { + return memorize(commit_entry); + } + + auto r = lua_->call, LScriptTranslator*, const CommitEntry&>( + memorize_callback_, this, commit_entry); + if (!r.ok()) { + auto e = r.get_err(); + LOG(ERROR) << "LScriptTranslator of " << name_space_ + << ": memorize_callback error(" << e.status << "): " << e.e; + return false; + } + return r.get(); + } + + void T::init_correction() { + if (auto* corrector = Corrector::Require("corrector")) { + Ticket ticket(engine_, name_space_); + corrector_.reset(corrector->Create(ticket)); + } + } + + void T::set_enable_correction(bool enable) { + if (enable_correction_ = enable && !corrector_) + init_correction(); + } + + bool T::update_entry(const DictEntry& entry, + int commits, const string& new_entory_prefix) { + if (user_dict_ && user_dict_->loaded()) + return user_dict_->UpdateEntry(entry, commits, new_entory_prefix); + + return false; + } + + bool T::reload_user_dict_disabling_patterns(an cl) { + return cl ? user_dict_disabling_patterns_.Load(cl) : false; + } + + static const luaL_Reg funcs[] = { + {NULL, NULL}, + }; + + static const luaL_Reg methods[] = { + {"query", WRAPMEM(T, Query)}, // string, segment + {"start_session", WRAPMEM(T, StartSession)}, + {"finish_session", WRAPMEM(T, FinishSession)}, + {"discard_session", WRAPMEM(T, DiscardSession)}, + WMEM(memorize), // delegate TableTransaltor::Momorize + WMEM(update_entry), // delegate UserDictionary::UpdateEntry + WMEM(reload_user_dict_disabling_patterns), + Set_WMEM(memorize_callback), // an callback function + {NULL, NULL}, + }; + + static const luaL_Reg vars_get[] = { + Get_WMEM(name_space), // string + Set_WMEM(memorize_callback), // an callback function + // ScriptTranslator member + Get_WMEM(max_homophones), // int + Get_WMEM(spelling_hints), // int + Get_WMEM(always_show_comments), // bool + Get_WMEM(enable_correction), // bool + //TranslatorOptions + Get_WMEM(delimiters), // string& + Get_WMEM(tag), // string + Get_WMEM(enable_completion), // bool + Get_WMEM(contextual_suggestions), // bool + Get_WMEM(strict_spelling), // bool + Get_WMEM(initial_quality), // double + Get_WMEM(preedit_formatter), // Projection& + Get_WMEM(comment_formatter), // Projection& + // Memory + Get_WMEM(dict), + Get_WMEM(user_dict), + {NULL, NULL}, + }; + + static const luaL_Reg vars_set[] = { + // ScriptTranslator member + Set_WMEM(max_homophones), // int + Set_WMEM(spelling_hints), // int + Set_WMEM(always_show_comments), // bool + Set_WMEM(enable_correction), // bool + // TranslatorOptions + Set_WMEM(delimiters), // string& + Set_WMEM(tag), // string + Set_WMEM(enable_completion), // bool + Set_WMEM(contextual_suggestions), // bool + Set_WMEM(strict_spelling), // bool + Set_WMEM(initial_quality), // double + Set_WMEM(preedit_formatter), // Projection& + Set_WMEM(comment_formatter), // Projection& + {NULL, NULL}, + }; + + void reg_Component(lua_State* L) { + lua_getglobal(L, "Component"); + if (lua_type(L, -1) != LUA_TTABLE) { + LOG(ERROR) << "table of _G[\"Component\"] not found."; + } else { + lua_pushcfunction(L, raw_make_translator); + lua_setfield(L, -2, "ScriptTranslator"); + } + lua_pop(L, 1); + } + +} // namespace ScriptTranslatorReg +} // namespace + +void LUAWRAPPER_LOCAL script_translator_init(lua_State* L) { + EXPORT(ScriptTranslatorReg, L); + ScriptTranslatorReg::reg_Component(L); +} diff --git a/src/table_translator.cc b/src/table_translator.cc new file mode 100644 index 0000000..b2647f4 --- /dev/null +++ b/src/table_translator.cc @@ -0,0 +1,223 @@ +/* + * table_translator.cc + * Copyright (C) 2023 Shewer Lu + * + * Distributed under terms of the MIT license. + */ + +#include +#include +#include +#include +#include + +#include "translator.h" + +using namespace rime; + +namespace { +namespace TableTranslatorReg { + + class LTableTranslator : public TableTranslator { + public: + LTableTranslator(const Ticket& ticket, Lua* lua) + : TableTranslator(ticket), lua_(lua) {}; + + virtual bool Memorize(const CommitEntry& commit_entry); + bool memorize(const CommitEntry& commit_entry); + bool update_entry(const DictEntry& index, + int commits, const string& new_entory_prefix); + + SET_(memorize_callback, an); + bool memorize_callback(); + + // TranslatorOptions + void set_contextual_suggestions(bool); + SET_(delimiters, string&); + SET_(preedit_formatter, Projection&); + SET_(comment_formatter, Projection&); + bool reload_user_dict_disabling_patterns(an); + + // TableTranslator member + ACCESS_(encode_commit_history, bool); + ACCESS_(max_phrase_length, int); + ACCESS_(max_homographs, int); + ACCESS_(enable_charset_filter, bool); + GET_(sentence_over_completion, bool); + void set_sentence_over_completion(bool); + GET_(enable_encoder, bool); + void set_enable_encoder(bool); + GET_(enable_sentence, bool); + void set_enable_sentence(bool); + + protected: + Lua* lua_; + an memorize_callback_; + bool init_poet(); + bool init_encoder(); + }; + + + using T = LTableTranslator; + + bool T::memorize_callback() { + return (memorize_callback_) ? true : false; + } + bool T::memorize(const CommitEntry& commit_entry) { + return TableTranslator::Memorize(commit_entry); + } + + bool T::Memorize(const CommitEntry& commit_entry) { + if (!memorize_callback_) { + return memorize(commit_entry); + } + + auto r = lua_->call, LTableTranslator*, const CommitEntry&>( + memorize_callback_, this, commit_entry); + if (!r.ok()) { + auto e = r.get_err(); + LOG(ERROR) << "LTableTranslator of " << name_space_ + << ": memorize_callback error(" << e.status << "): " << e.e; + return false; + } + return r.get(); + } + + bool T::update_entry(const DictEntry& entry, + int commits, const string& new_entory_prefix) { + if (user_dict_ && user_dict_->loaded()) + return user_dict_->UpdateEntry(entry, commits, new_entory_prefix); + + return false; + } + + // enable_encoder + bool T::init_encoder() { + if (!user_dict_) + return false; + encoder_.reset(new UnityTableEncoder(user_dict_.get())); + Ticket ticket(engine_, name_space_); + encoder_->Load(ticket); + if (!encoder_) { + LOG(WARNING) << "init encoder failed"; + return false; + } + return true; + } + + void T::set_enable_encoder(bool enable) { + if (enable_encoder_ = enable && user_dict_ && !encoder_){ + init_encoder(); + } + } + // enable sentence contextual + bool T::init_poet() { + Config* config = engine_->schema()->config(); + poet_.reset(new Poet(language(), config, Poet::LeftAssociateCompare)); + if (!poet_) { + LOG(WARNING) << "init poet failed"; + return false; + } + return true; + } + + void T::set_enable_sentence(bool enable) { + if (enable_sentence_ = enable && !poet_) + init_poet(); + } + + void T::set_sentence_over_completion(bool enable) { + if (sentence_over_completion_ = enable && !poet_ ) + init_poet(); + } + + void T::set_contextual_suggestions(bool enable) { + if (contextual_suggestions_ = enable && !poet_) + init_poet(); + } + bool T::reload_user_dict_disabling_patterns(an cl) { + return cl ? user_dict_disabling_patterns_.Load(cl) : false; + } + + static const luaL_Reg funcs[] = { + {NULL, NULL}, + }; + + static const luaL_Reg methods[] = { + {"query", WRAPMEM(T, Query)}, // string, segment + {"start_session", WRAPMEM(T, StartSession)}, + {"finish_session", WRAPMEM(T, FinishSession)}, + {"discard_session", WRAPMEM(T, DiscardSession)}, + WMEM(memorize), // delegate TableTransaltor::Momorize + WMEM(update_entry), // delegate UserDictionary::UpdateEntry + WMEM(reload_user_dict_disabling_patterns), + Set_WMEM(memorize_callback), // an callback function + {NULL, NULL}, + }; + + static const luaL_Reg vars_get[] = { + // class translator member + Get_WMEM(name_space), // string + Get_WMEM(memorize_callback), // an callback function + // TabletTranslator member + Get_WMEM(enable_charset_filter), // bool + Get_WMEM(enable_encoder), // bool + Get_WMEM(enable_sentence), // bool + Get_WMEM(sentence_over_completion), // bool + Get_WMEM(encode_commit_history), // int + Get_WMEM(max_phrase_length), // int + Get_WMEM(max_homographs), // bool + // TranslatorOptions + Get_WMEM(delimiters), // string& + Get_WMEM(tag), // string + Get_WMEM(enable_completion), // bool + Get_WMEM(contextual_suggestions), // bool + Get_WMEM(strict_spelling), // bool + Get_WMEM(initial_quality), // double + Get_WMEM(preedit_formatter), // Projection& + Get_WMEM(comment_formatter), // Projection& + // Memory + Get_WMEM(dict), + Get_WMEM(user_dict), + {NULL, NULL}, + }; + + static const luaL_Reg vars_set[] = { + // TableTranslator member + Set_WMEM(enable_charset_filter), // bool + Set_WMEM(enable_encoder), // bool + Set_WMEM(enable_sentence), // bool + Set_WMEM(sentence_over_completion), // bool + Set_WMEM(encode_commit_history), // bool + Set_WMEM(max_phrase_length), // int + Set_WMEM(max_homographs), // int + // TranslatorOptions + Set_WMEM(delimiters), // string& + Set_WMEM(tag), // string + Set_WMEM(enable_completion), // bool + Set_WMEM(contextual_suggestions), // bool + Set_WMEM(strict_spelling), // bool + Set_WMEM(initial_quality), // double + Set_WMEM(preedit_formatter), // Projection& + Set_WMEM(comment_formatter), // Projection& + {NULL, NULL}, + }; + + void reg_Component(lua_State* L) { + lua_getglobal(L, "Component"); + if (lua_type(L, -1) != LUA_TTABLE) { + LOG(ERROR) << "table of _G[\"Component\"] not found."; + } else { + lua_pushcfunction(L, raw_make_translator); + lua_setfield(L, -2, "TableTranslator"); + } + lua_pop(L, 1); + } + +} // namespace TableTranslatorReg +} // namespace + +void LUAWRAPPER_LOCAL table_translator_init(lua_State* L) { + EXPORT(TableTranslatorReg, L); + TableTranslatorReg::reg_Component(L); +} diff --git a/src/translator.h b/src/translator.h new file mode 100644 index 0000000..d5bd9b2 --- /dev/null +++ b/src/translator.h @@ -0,0 +1,69 @@ +/* + * script_translator.h + * Copyright (C) 2023 Shewer Lu + * + * Distributed under terms of the MIT license. + */ + +#ifndef _LUA_TRANSLATOR_H +#define _LUA_TRANSLATOR_H + +#include +#include +#include +#include +#include +#include +#include "lib/lua_export_type.h" + +#define SET_(name, type) \ + void set_##name(const type data) { \ + name##_ = data; \ + } +#define GET_(name, type) \ + type name() { \ + return name##_; \ + } +#define ACCESS_(name, type) \ + SET_(name, type); \ + GET_(name, type) + +#define Set_WMEM(name) \ + { #name, WRAPMEM(T, set_##name) } +#define WMEM(name) \ + { #name, WRAPMEM(T, name) } +#define Get_WMEM(name) \ + { #name, WRAPMEM(T, name) } + +#if LUA_VERSION_NUM < 502 +#define lua_absindex(L, i) abs_index((L), (i)) +#endif + + +template +int raw_make_translator(lua_State* L){ + int n = lua_gettop(L); + if (3 > n || 4 < n) + return 0; + + C_State C; + rime::Ticket ticket( + LuaType::todata(L, 1), + LuaType::todata(L, -2, &C), + LuaType::todata(L, -1, &C) + ); + DLOG(INFO) << "check Ticket:" << ticket.klass << "@" <::todata(L, 2) ); //overwrite schema + Lua* lua= Lua::from_state(L); + rime::an obj = rime::New(ticket, lua); + if (obj) { + LuaType>::pushdata(L, obj); + return 1; + } + else { + //LOG(ERROR) << "error creating " << typeid(O).name() << ": '" << ticket.klass << "'"; + return 0; + } +}; +#endif /* !_LUA_TRANSLATOR_H */ diff --git a/src/types_ext.cc b/src/types_ext.cc index 66670ec..a69d033 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -16,11 +16,9 @@ #include "lib/lua_export_type.h" #include "optional.h" - #include using namespace rime; - namespace { template using void_t = void; @@ -253,11 +251,11 @@ namespace UserDbReg{ an make_leveldb(const string& db_name) { return make(db_name, "userdb"); } - + an make_tabledb(const string& db_name) { return make(db_name, "plain_userdb"); } - + optional fetch(an t, const string& key) { string res; if ( t->Fetch(key,&res) ) @@ -274,7 +272,7 @@ namespace UserDbReg{ } an Query(T &t, const string& key) { return t.Query(key); } - + static const luaL_Reg funcs[] = { {"UserDb", WRAP(make)},// an LevelDb( db_file, db_name) {"LevelDb", WRAP(make_leveldb)},// an LevelDb( db_file, db_name) @@ -290,7 +288,7 @@ namespace UserDbReg{ {"fetch", WRAP(fetch)}, // fetch(key) return value {"update", WRAP(Update)}, // update(key,value) return bool {"erase", WRAP(Erase)}, // erase(key) return bool - + {"loaded",WRAPMEM(T, loaded)}, {"disable", WRAPMEM(T, disable)}, {"enable", WRAPMEM(T, enable)}, @@ -343,6 +341,7 @@ namespace ComponentReg{ } }; + static const luaL_Reg funcs[] = { {"Processor", raw_create

}, {"Segmentor" , raw_create}, @@ -360,6 +359,9 @@ namespace ComponentReg{ } +void table_translator_init(lua_State *L); +void script_translator_init(lua_State *L); + void LUAWRAPPER_LOCAL types_ext_init(lua_State *L) { EXPORT(ProcessorReg, L); EXPORT(SegmentorReg, L); @@ -369,4 +371,7 @@ void LUAWRAPPER_LOCAL types_ext_init(lua_State *L) { EXPORT(DbAccessorReg, L); EXPORT(UserDbReg, L); ComponentReg::init(L); + // add LtableTranslator ScriptTranslator in Component + table_translator_init(L); + script_translator_init(L); } From cf2c5722b9d9f2f05e1ead1b9959d9a7f84f06c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=85=E6=88=8E=E6=B0=8F?= Date: Tue, 6 Feb 2024 22:43:00 +0800 Subject: [PATCH 32/47] follow up librime#806, avoid path-string conversion (#310) * follow up librime#806, avoid extra conversion Avoid using `RimeGet*Dir()` from `rime_api`. They return native path on Windows and coding conversion is incurred. When passing path from lua to librime class, use either rime::path object or UTF-8 encoded string. * replace RimeApi::get_*_dir with Deployer's getters * compat old librime --------- Co-authored-by: Chunhui He --- src/modules.cc | 33 +++++++++++++++++-- src/opencc.cc | 61 +++++++++++++++++++++++++---------- src/types.cc | 82 ++++++++++++++++++++++++++++++------------------ src/types_ext.cc | 19 ++++++++++- 4 files changed, 145 insertions(+), 50 deletions(-) diff --git a/src/modules.cc b/src/modules.cc index 2e84459..a4ed5e0 100644 --- a/src/modules.cc +++ b/src/modules.cc @@ -2,6 +2,7 @@ #include #include #include +#include #include "lib/lua_templates.h" #include "lua_gears.h" @@ -16,9 +17,37 @@ static bool file_exists(const char *fname) noexcept { return false; } +namespace { +template using void_t = void; + +template +struct COMPAT { + static std::string get_shared_data_dir() { + return std::string(rime_get_api()->get_shared_data_dir()); + } + + static std::string get_user_data_dir() { + return std::string(rime_get_api()->get_user_data_dir()); + } +}; + +template +struct COMPAT().user_data_dir.string())>> { + static std::string get_shared_data_dir() { + // path::string() returns native encoding on Windows + return rime::Service::instance().deployer().shared_data_dir.string(); + } + + static std::string get_user_data_dir() { + return rime::Service::instance().deployer().user_data_dir.string(); + } +}; +} + static void lua_init(lua_State *L) { - const auto user_dir = std::string(RimeGetUserDataDir()); - const auto shared_dir = std::string(RimeGetSharedDataDir()); + + const auto user_dir = COMPAT::get_user_data_dir(); + const auto shared_dir = COMPAT::get_shared_data_dir(); types_init(L); lua_getglobal(L, "package"); diff --git a/src/opencc.cc b/src/opencc.cc index 17c9b90..16a5647 100644 --- a/src/opencc.cc +++ b/src/opencc.cc @@ -13,8 +13,8 @@ #include #include #include -#include #include +#include #include "lib/lua_export_type.h" #include "optional.h" @@ -28,8 +28,8 @@ namespace { class Opencc { public: - //static shared_ptr create(const string &config_path); - Opencc(const string& config_path); + //static shared_ptr create(const path &config_path); + Opencc(const string& utf8_config_path); bool ConvertWord(const string& text, vector* forms); bool RandomConvertText(const string& text, string* simplified); bool ConvertText(const string& text, string* simplified); @@ -43,9 +43,10 @@ class Opencc { opencc::DictPtr dict_; }; -Opencc::Opencc(const string& config_path) { +Opencc::Opencc(const string& utf8_config_path) { opencc::Config config; - converter_ = config.NewFromFile(config_path); + // OpenCC accepts UTF-8 encoded path. + converter_ = config.NewFromFile(utf8_config_path); const list conversions = converter_->GetConversionChain()->GetConversions(); dict_ = conversions.front()->GetDict(); @@ -115,23 +116,49 @@ vector Opencc::convert_word(const string& text){ namespace OpenccReg { using T = Opencc; - optional make(const string &filename) { - string user_path( RimeGetUserDataDir()); - string shared_path(RimeGetSharedDataDir()); - try{ - return T(user_path + "/opencc/" + filename); + template using void_t = void; + + template + struct COMPAT { + static optional make(const string &filename) { + auto user_path = string(rime_get_api()->get_user_data_dir()); + auto shared_path = string(rime_get_api()->get_shared_data_dir()); + try{ + return T(user_path + "/opencc/" + filename); + } + catch(...) { + try{ + return T(shared_path + "/opencc/" + filename); + } + catch(...) { + LOG(ERROR) << " [" << user_path << "|" << shared_path << "]/opencc/" + << filename << ": File not found or InvalidFormat"; + return {}; + } + } } - catch(...) { + }; + + template + struct COMPAT().user_data_dir.string())>> { + static optional make(const string &filename) { + path user_path = Service::instance().deployer().user_data_dir; + path shared_path = Service::instance().deployer().shared_data_dir; try{ - return T(shared_path + "/opencc/" + filename); + return T((user_path / "opencc" / filename).u8string()); } catch(...) { - LOG(ERROR) << " [" << user_path << "|" << shared_path << "]/opencc/" - << filename << ": File not found or InvalidFormat"; - return {}; + try{ + return T((shared_path / "opencc" / filename).u8string()); + } + catch(...) { + LOG(ERROR) << " [" << user_path << "|" << shared_path << "]/opencc/" + << filename << ": File not found or InvalidFormat"; + return {}; + } } } - } + }; optional> convert_word(T &t,const string &s) { vector res; @@ -141,7 +168,7 @@ namespace OpenccReg { } static const luaL_Reg funcs[] = { - {"Opencc",WRAP(make)}, + {"Opencc",WRAP(COMPAT::make)}, { NULL, NULL }, }; diff --git a/src/types.cc b/src/types.cc index 21fdd9f..1f4fe74 100644 --- a/src/types.cc +++ b/src/types.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -15,9 +16,9 @@ #include #include #include +#include #include #include "lua_gears.h" -#include #include #include "lib/lua_export_type.h" @@ -29,6 +30,47 @@ using namespace rime; namespace { +template using void_t = void; + +template +struct COMPAT { + // fallback version if librime is old + static an new_ReverseDb(const std::string &file) { + return New(std::string(RimeGetUserDataDir()) + "/" + file); + } + + static string get_shared_data_dir() { + return string(rime_get_api()->get_shared_data_dir()); + } + + static string get_user_data_dir() { + return string(rime_get_api()->get_user_data_dir()); + } + + static string get_sync_dir() { + return string(rime_get_api()->get_sync_dir()); + } +}; + +template +struct COMPAT().user_data_dir.string())>> { + static an new_ReverseDb(const std::string &file) { + return New(Service::instance().deployer().user_data_dir / file); + } + + static string get_shared_data_dir() { + return Service::instance().deployer().shared_data_dir.string(); + } + + static string get_user_data_dir() { + return Service::instance().deployer().user_data_dir.string(); + } + + static string get_sync_dir() { + return Service::instance().deployer().sync_dir.string(); + } +}; + //--- wrappers for Segment namespace SegmentReg { using T = Segment; @@ -308,7 +350,7 @@ namespace ReverseDbReg { using T = ReverseDb; an make(const string &file) { - an db = New(string(RimeGetUserDataDir()) + "/" + file); + an db = COMPAT::new_ReverseDb(file); db->Load(); return db; } @@ -1844,43 +1886,23 @@ namespace KeySequenceReg { namespace RimeApiReg { string get_rime_version() { - RimeApi* rime = rime_get_api(); - return string(rime->get_version()); - } - - string get_shared_data_dir() { - RimeApi* rime = rime_get_api(); - return string(rime->get_shared_data_dir()); - } - - string get_user_data_dir() { - RimeApi* rime = rime_get_api(); - return string(rime->get_user_data_dir()); - } - - string get_sync_dir() { - RimeApi* rime = rime_get_api(); - return string(rime->get_sync_dir()); + return string(rime_get_api()->get_version()); } string get_distribution_name(){ - Deployer &deployer(Service::instance().deployer()); - return deployer.distribution_name; + return Service::instance().deployer().distribution_name; } string get_distribution_code_name(){ - Deployer &deployer(Service::instance().deployer()); - return deployer.distribution_code_name; + return Service::instance().deployer().distribution_code_name; } string get_distribution_version(){ - Deployer &deployer(Service::instance().deployer()); - return deployer.distribution_version; + return Service::instance().deployer().distribution_version; } string get_user_id(){ - Deployer &deployer(Service::instance().deployer()); - return deployer.user_id; + return Service::instance().deployer().user_id; } // boost::regex api @@ -1912,9 +1934,9 @@ namespace RimeApiReg { static const luaL_Reg funcs[]= { { "get_rime_version", WRAP(get_rime_version) }, - { "get_shared_data_dir", WRAP(get_shared_data_dir) }, - { "get_user_data_dir", WRAP(get_user_data_dir) }, - { "get_sync_dir", WRAP(get_sync_dir) }, + { "get_shared_data_dir", WRAP(COMPAT::get_shared_data_dir) }, + { "get_user_data_dir", WRAP(COMPAT::get_user_data_dir) }, + { "get_sync_dir", WRAP(COMPAT::get_sync_dir) }, { "get_distribution_name", WRAP(get_distribution_name) }, { "get_distribution_code_name", WRAP(get_distribution_code_name) }, { "get_distribution_version", WRAP(get_distribution_version) }, diff --git a/src/types_ext.cc b/src/types_ext.cc index a69d033..99e351d 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -38,6 +38,18 @@ struct COMPAT().name_space())>> { } }; +// fallback version of file_path() if librime is old +template struct void_t1 { using t = int; }; +template().file_name())>::t = 0> +std::string get_UserDb_file_path_string(const T &t) { + return t.file_name(); +} + +template().file_path())>::t = 0> +std::string get_UserDb_file_path_string(const T &t) { + return t.file_path().string(); +} + namespace ProcessorReg{ using T = Processor; @@ -263,6 +275,11 @@ namespace UserDbReg{ return {}; } + string file_name(const T& t) { + // returns ANSI encoded string on Windows + return t.file_path().string(); + } + bool Open(T &t) { return t.Open(); } bool Close(T &t) { return t.Close(); } bool OpenReadOnly(T &t) { return t.OpenReadOnly(); } @@ -300,7 +317,7 @@ namespace UserDbReg{ {"read_only",WRAPMEM(T, readonly)}, {"disabled",WRAPMEM(T, disabled)}, {"name", WRAPMEM(T, name)}, - {"file_name", WRAPMEM(T, file_name)}, + {"file_name", WRAP(get_UserDb_file_path_string)}, { NULL, NULL }, }; From 7f3eca2ce659fc2401b8acb52bd2182b433e12b1 Mon Sep 17 00:00:00 2001 From: hchunhui Date: Tue, 6 Feb 2024 22:44:12 +0800 Subject: [PATCH 33/47] fix: reduce peak memory usage (#308) --- src/lib/lua.cc | 4 ++++ src/lib/lua.h | 2 ++ src/lua_gears.cc | 4 ++++ src/lua_gears.h | 2 ++ 4 files changed, 12 insertions(+) diff --git a/src/lib/lua.cc b/src/lib/lua.cc index c68fdc1..ced2a24 100644 --- a/src/lib/lua.cc +++ b/src/lib/lua.cc @@ -264,6 +264,10 @@ std::shared_ptr Lua::getglobal(const std::string &v) { return o; } +void Lua::gc() { + lua_gc(L_, LUA_GCCOLLECT, 0); +} + LuaObj::LuaObj(lua_State *L, int i) : L_(L) { lua_pushvalue(L, i); id_ = luaL_ref(L, LUA_REGISTRYINDEX); diff --git a/src/lib/lua.h b/src/lib/lua.h index 6eedd87..c2d00c4 100644 --- a/src/lib/lua.h +++ b/src/lib/lua.h @@ -34,6 +34,8 @@ class Lua { std::shared_ptr newthreadx(lua_State *L, int nargs); + void gc(); + template std::shared_ptr newthread(I ... input); diff --git a/src/lua_gears.cc b/src/lua_gears.cc index 4e70092..9e1f704 100644 --- a/src/lua_gears.cc +++ b/src/lua_gears.cc @@ -23,6 +23,10 @@ bool LuaTranslation::Next() { } } +LuaTranslation::~LuaTranslation() { + lua_->gc(); +} + static std::vector split_string(const std::string& str, const std::string& delimiter) { std::vector result; size_t pos = 0; diff --git a/src/lua_gears.h b/src/lua_gears.h index d763970..8c44c6f 100644 --- a/src/lua_gears.h +++ b/src/lua_gears.h @@ -24,6 +24,8 @@ class LuaTranslation : public Translation { return c_; } + virtual ~LuaTranslation(); + private: Lua *lua_; an c_; From 3912404b8812a745f06f9093dbec15ed9065d786 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Sun, 25 Feb 2024 12:35:30 +0800 Subject: [PATCH 34/47] revert user_lookup() boot dict_lookup and add useriter_lookup() dictiter_lookup() (#305) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * changed member type on LuaMemory iter, utermember LuaMemory Signed-off-by: shewer * add DictionaryReg UserDictionaryReg Signed-off-by: shewer * 增加 SentenceReg ,update_candidate , :update_entry 增加leng_name 用於檢查 lasguage Signed-off-by: shewer * revert user_lookup() boot dict_lookup and add useriter_lookup() dictiter_lookup() Signed-off-by: shewer * 增加 Translation.exhausted UserDictEntryIterator DictEntryIterator get_vars( exhausted, size) Signed-off-by: shewer --------- Signed-off-by: shewer --- src/types.cc | 376 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 307 insertions(+), 69 deletions(-) diff --git a/src/types.cc b/src/types.cc index 1f4fe74..877477d 100644 --- a/src/types.cc +++ b/src/types.cc @@ -147,6 +147,8 @@ namespace CandidateReg { using T = Candidate; string dynamic_type(T &c) { + if (dynamic_cast(&c)) + return "Sentence"; if (dynamic_cast(&c)) return "Phrase"; if (dynamic_cast(&c)) @@ -338,6 +340,7 @@ namespace TranslationReg { }; static const luaL_Reg vars_get[] = { + {"exhausted", WRAPMEM(T, exhausted)}, { NULL, NULL }, }; @@ -1572,34 +1575,221 @@ namespace CodeReg { { NULL, NULL }, }; } + +namespace DictionaryReg { + using T = Dictionary; + using I = DictEntryIterator; + using D = DictEntry; + + an lookup_words(T& t, const string& code, bool predictive , size_t limit) { + an ret=New(); + t.LookupWords(ret.get(),code, predictive, limit); + return ret; + } + + vector decode(T& t, const Code& code) { + vector ret; + t.Decode(code, &ret); + return ret; + } + + static const luaL_Reg funcs[] = { + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + { "lookup_words", WRAP(lookup_words)}, + //{ "lookup", WRAPMEM(T, Lookup)}, + { "decode", WRAP(decode)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + { "name", WRAPMEM(T, name)}, + { "loaded", WRAPMEM(T, loaded)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { NULL, NULL }, + }; +} + +namespace UserDictionaryReg { + using T = UserDictionary; + using I = UserDictEntryIterator; + using D = DictEntry; + + an lookup_words(T& t, const string& code, bool predictive , size_t limit) { + an ret=New(); + t.LookupWords(ret.get(),code, predictive, limit); + return ret; + } + bool update_entry(T& t, const D& entry, int commits, const string& prefix, + const string lang_name) { + return (lang_name == t.name()) ? t.UpdateEntry(entry, commits, prefix) : false; + } + + static const luaL_Reg funcs[] = { + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + { "lookup_words", WRAP(lookup_words)}, + //{ "lookup", WRAPMEM(T, Lookup)}, + { "update_entry", WRAP(update_entry)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + { "name", WRAPMEM(T, name)}, + { "loaded", WRAPMEM(T, loaded)}, + { "tick", WRAPMEM(T, tick)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { NULL, NULL }, + }; +} + +namespace DictEntryIteratorReg { + using T = DictEntryIterator; + using D = DictEntry; + + optional> Next(T& t) { + if ( t.exhausted()) { + return {}; + } + an ret = t.Peek(); + t.Next(); + return ret; + } + + int raw_iter(lua_State* L) { + int n = lua_gettop(L); + if (n>=1) { // :iter() + lua_pushcfunction(L, WRAP(Next)); + lua_insert(L, 1); + return 2; + } + return 0; + } + + static const luaL_Reg funcs[] = { + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + {"iter", raw_iter}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + {"exhausted", WRAPMEM(T, exhausted)}, + {"size", WRAPMEM(T, entry_count)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { NULL, NULL }, + }; + +} + +namespace UserDictEntryIteratorReg { + using T = UserDictEntryIterator; + using D = DictEntry; + + optional> Next(T& t) { + if ( t.exhausted()) { + return {}; + } + an ret = t.Peek(); + t.Next(); + return ret; + } + + int raw_iter(lua_State* L) { + int n = lua_gettop(L); + if (n>=1) { // :iter() + lua_pushcfunction(L, WRAP(Next)); + lua_insert(L, 1); + return 2; + } + return 0; + } + + static const luaL_Reg funcs[] = { + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + {"iter", raw_iter}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + {"exhausted", WRAPMEM(T, exhausted)}, + {"size", WRAPMEM(T, cache_size)}, + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { NULL, NULL }, + }; +} namespace MemoryReg { class LuaMemory : public Memory { an memorize_callback; Lua *lua_; public: - using Memory::Memory; - DictEntryIterator iter; - UserDictEntryIterator uter; + an iter; + an uter; - LuaMemory(Lua *lua, const Ticket& ticket) - : lua_(lua), Memory(ticket) {} + bool dictLookup(const string& str_code, + bool predictive, size_t expand_search_limit); + + bool userLookup(const string& input, const bool isExpand); - virtual bool Memorize(const CommitEntry&); + an dictiterLookup(const string& str_code, + bool predictive, size_t expand_search_limit); + + an useriterLookup(const string& input, const bool isExpand); + + vector decode(const Code& code); + bool update_userdict(const DictEntry& entry, const int commits, + const string& new_entry_prefix); + + bool update_entry(const DictEntry& entry, const int commits, + const string& new_entry_prefix, const string& lang_string); + + bool update_candidate(const an cand, const int commits); + const string lang_name() { return language_->name(); }; + virtual bool Memorize(const CommitEntry& entry); + + LuaMemory(const Ticket& ticket, Lua* lua) + : lua_(lua), Memory(ticket) {} void memorize(an func) { memorize_callback = func; } - void clearDict() { - iter = DictEntryIterator(); + iter.reset(); } void clearUser() { - uter = UserDictEntryIterator(); + uter.reset(); } }; - using T = LuaMemory; - bool MemoryReg::LuaMemory::Memorize(const CommitEntry& commit_entry) { + vector LuaMemory::decode(const Code& code) { + vector res; + if (dict_ && dict_->loaded()) + dict_->Decode(code,&res); + return res; + } + + bool LuaMemory::Memorize(const CommitEntry& commit_entry) { if (!memorize_callback) return false; @@ -1612,8 +1802,78 @@ namespace MemoryReg { return r.get(); } + bool LuaMemory::dictLookup(const string& input, const bool isExpand, size_t limit) { + iter = New();// t= New(); + limit = limit == 0 ? 0xffffffffffffffff : limit; + if (dict_ && dict_->loaded()) { + return dict_->LookupWords(iter.get(), input, isExpand, limit) > 0; + } + return false; + } + + bool LuaMemory::userLookup(const string& input, const bool isExpand) { + uter = New(); + if (user_dict_ && user_dict_->loaded()) { + return user_dict_->LookupWords(uter.get(), input, isExpand) > 0; + } + return false; + } + + an LuaMemory::dictiterLookup(const string& input, const bool isExpand, size_t limit) { + dictLookup(input, isExpand, limit); + return iter; + } + + an LuaMemory::useriterLookup(const string& input, const bool isExpand) { + userLookup(input, isExpand); + return uter; + } + + + bool LuaMemory::update_userdict(const DictEntry& entry, const int commits, + const string& new_entry_prefix) { + if (user_dict_ && user_dict_->loaded()) + return user_dict_->UpdateEntry(entry, commits, new_entry_prefix); + + return false; + } + + bool LuaMemory::update_entry(const DictEntry& entry, const int commits, + const string& new_entry_prefix, const string& lang_name) + { + if (user_dict_ && user_dict_->loaded() && lang_name == language_->name()) + return user_dict_->UpdateEntry(entry, commits, new_entry_prefix); + + return false; + } + + bool LuaMemory::update_candidate(const an cand, const int commits) { + if (!user_dict_ || !user_dict_->loaded()) + return false; + + bool res = false; + vector> cands = Candidate::GetGenuineCandidates(cand); + for (auto cand : cands) { + if (auto c = std::dynamic_pointer_cast(cand)) { + if (*language_ == *c->language()) { + for (auto entry : c->components()) { + res |= user_dict()->UpdateEntry(c->entry(), commits); + }//components + } + } + else if (auto c = std::dynamic_pointer_cast(cand)) { + res |= (*language_ == *c->language()) && + user_dict()->UpdateEntry(c->entry(), commits); + } + }// cands + return res; + } + // XXX: Currently the WRAP macro is not generic enough, // so that we need a raw function to get the lua state / parse variable args. + + using T = LuaMemory; + int raw_make(lua_State *L) { // TODO: fix the memory leak C_State C; @@ -1629,63 +1889,36 @@ namespace MemoryReg { if (3 <= n) translatorTicket.name_space = LuaType::todata(L, 3, &C); - an memoli = New(lua, translatorTicket); + an memoli = New(translatorTicket, lua); LuaType>::pushdata(L, memoli); return 1; } - bool dictLookup(T& memory, const string& input, const bool isExpand,size_t limit) { - memory.clearDict(); - limit = limit == 0 ? 0xffffffffffffffff : limit; - if (auto dict = memory.dict()) - return dict->LookupWords(&memory.iter, input, isExpand, limit) > 0; - else - return false; - } - - optional> dictNext(T& memory) { - if (memory.iter.exhausted()) { - return {}; - } - an ret = memory.iter.Peek(); - memory.iter.Next(); - return ret; - } - - bool userLookup(T& memory, const string& input, const bool isExpand) { - memory.clearUser(); - if (auto dict = memory.user_dict()) - return dict->LookupWords(&memory.uter, input, isExpand) > 0; - else - return false; - } - - optional> userNext(T& memory) { - if (memory.uter.exhausted()) { - return {}; - } - an ret = memory.uter.Peek(); - memory.uter.Next(); - return ret; - } - - bool updateToUserdict(T& memory, const DictEntry& entry, const int commits, const string& new_entry_prefix) { - if (auto dict = memory.user_dict()) - return dict->UpdateEntry(entry, commits, new_entry_prefix); - else - return false; - } + // :iter_user([func[,...]]) return next_func, entryiterator + // return next_entry, enter_iter[,func[, ...]] int raw_iter_user(lua_State* L) { - lua_pushcfunction(L, WRAP(userNext)); - lua_pushvalue(L, 1); - return 2; + an t = LuaType>::todata(L, 1); + LuaType>::pushdata(L, t->uter); + lua_replace(L, 1); + lua_getfield(L, 1, "iter"); + lua_insert(L, 1); + if (lua_pcall(L, lua_gettop(L) -1 , 2,0) != LUA_OK) + return 0; + return lua_gettop(L); } + // :iter_user([func[,...]]) return next_func, entryiterator + // return next_entry, enter_iter[,func[, ...]] int raw_iter_dict(lua_State* L) { - lua_pushcfunction(L, WRAP(dictNext)); - lua_pushvalue(L, 1); - return 2; + an t = LuaType>::todata(L, 1); + LuaType>::pushdata(L, t->iter); + lua_replace(L, 1); + lua_getfield(L, 1, "iter"); + lua_insert(L, 1); + if (lua_pcall(L, lua_gettop(L) -1 , 2,0) != LUA_OK) + return 0; + return lua_gettop(L); } static const luaL_Reg funcs[] = { @@ -1693,24 +1926,25 @@ namespace MemoryReg { {NULL, NULL}, }; - std::vector decode(T& memory, Code& code) { - std::vector res; - if (auto dict = memory.dict()) - dict->Decode(code,&res); - return res; - } static const luaL_Reg methods[] = { - { "dict_lookup", WRAP(dictLookup)}, - { "user_lookup", WRAP(userLookup)}, + { "dict_lookup", WRAPMEM(T::dictLookup)}, // bool + { "user_lookup", WRAPMEM(T::userLookup)}, // bool + { "dictiter_lookup", WRAPMEM(T::dictiterLookup)}, // iter + { "useriter_lookup", WRAPMEM(T::useriterLookup)}, // iter { "memorize", WRAPMEM(T::memorize)}, - { "decode", WRAP(decode)}, + { "decode", WRAPMEM(T::decode)}, { "iter_dict", raw_iter_dict}, { "iter_user", raw_iter_user}, - { "update_userdict", WRAP(updateToUserdict)}, + { "update_userdict", WRAPMEM(T::update_userdict)}, + { "update_entry", WRAPMEM(T::update_entry)}, + { "update_candidate", WRAPMEM(T::update_candidate)}, {NULL, NULL}, }; static const luaL_Reg vars_get[] = { + { "lang_name", WRAPMEM(T, lang_name)}, + { "dict", WRAPMEM(T, dict)}, + { "user_dict", WRAPMEM(T, user_dict)}, {NULL, NULL}, }; @@ -2020,6 +2254,10 @@ void types_init(lua_State *L) { EXPORT(KeyEventNotifierReg, L); EXPORT(ConnectionReg, L); EXPORT(MemoryReg, L); + EXPORT(DictionaryReg, L); + EXPORT(UserDictionaryReg, L); + EXPORT(DictEntryIteratorReg, L); + EXPORT(UserDictEntryIteratorReg, L); EXPORT(DictEntryReg, L); EXPORT(CodeReg, L); EXPORT(CommitEntryReg, L); From 26d74b9559ed8f0314a4a1b40dab954b533d7fe2 Mon Sep 17 00:00:00 2001 From: groverlynn Date: Sun, 25 Feb 2024 05:38:40 +0100 Subject: [PATCH 35/47] silence warning (#315) --- src/script_translator.cc | 2 +- src/table_translator.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/script_translator.cc b/src/script_translator.cc index e604955..467f68a 100644 --- a/src/script_translator.cc +++ b/src/script_translator.cc @@ -85,7 +85,7 @@ namespace ScriptTranslatorReg { } void T::set_enable_correction(bool enable) { - if (enable_correction_ = enable && !corrector_) + if ((enable_correction_ = enable && !corrector_)) init_correction(); } diff --git a/src/table_translator.cc b/src/table_translator.cc index b2647f4..ef6722b 100644 --- a/src/table_translator.cc +++ b/src/table_translator.cc @@ -106,7 +106,7 @@ namespace TableTranslatorReg { } void T::set_enable_encoder(bool enable) { - if (enable_encoder_ = enable && user_dict_ && !encoder_){ + if ((enable_encoder_ = enable && user_dict_ && !encoder_)) { init_encoder(); } } @@ -122,17 +122,17 @@ namespace TableTranslatorReg { } void T::set_enable_sentence(bool enable) { - if (enable_sentence_ = enable && !poet_) + if ((enable_sentence_ = enable && !poet_)) init_poet(); } void T::set_sentence_over_completion(bool enable) { - if (sentence_over_completion_ = enable && !poet_ ) + if ((sentence_over_completion_ = enable && !poet_ )) init_poet(); } void T::set_contextual_suggestions(bool enable) { - if (contextual_suggestions_ = enable && !poet_) + if ((contextual_suggestions_ = enable && !poet_)) init_poet(); } bool T::reload_user_dict_disabling_patterns(an cl) { From b2ad34cc8c639b7bb1bc6abcde08621b57e3e055 Mon Sep 17 00:00:00 2001 From: groverlynn Date: Sun, 25 Feb 2024 05:39:36 +0100 Subject: [PATCH 36/47] chained conversion (#316) --- src/opencc.cc | 99 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 25 deletions(-) diff --git a/src/opencc.cc b/src/opencc.cc index 16a5647..1cf06c9 100644 --- a/src/opencc.cc +++ b/src/opencc.cc @@ -52,7 +52,6 @@ Opencc::Opencc(const string& utf8_config_path) { dict_ = conversions.front()->GetDict(); } - bool Opencc::ConvertText(const string& text, string* simplified) { if (converter_ == nullptr) return false; *simplified = converter_->Convert(text); @@ -60,40 +59,90 @@ bool Opencc::ConvertText(const string& text, string* simplified) { } bool Opencc::ConvertWord(const string& text, vector* forms) { - if (dict_ == nullptr) return false; - opencc::Optional item = dict_->Match(text); - if (item.IsNull()) { - // Match not found - return false; - } else { - const opencc::DictEntry* entry = item.Get(); - for (auto&& value : entry->Values()) { - forms->push_back(std::move(value)); + if (converter_ == nullptr) return false; + const list conversions = + converter_->GetConversionChain()->GetConversions(); + vector original_words{text}; + bool matched = false; + for (auto conversion : conversions) { + opencc::DictPtr dict = conversion->GetDict(); + if (dict == nullptr) return false; + set word_set; + vector converted_words; + for (const auto& original_word : original_words) { + opencc::Optional item = + dict->Match(original_word); + if (item.IsNull()) { + // No exact match, but still need to convert partially matched + std::ostringstream buffer; + for (const char* wstr = original_word.c_str(); *wstr != '\0';) { + opencc::Optional matched = + dict->MatchPrefix(wstr); + size_t matched_length; + if (matched.IsNull()) { + matched_length = opencc::UTF8Util::NextCharLength(wstr); + buffer << opencc::UTF8Util::FromSubstr(wstr, matched_length); + } else { + matched_length = matched.Get()->KeyLength(); + buffer << matched.Get()->GetDefault(); + } + wstr += matched_length; + } + const string& converted_word = buffer.str(); + // Even if current dictionary doesn't convert the word + // (converted_word == original_word), we still need to keep it for + // subsequent dicts in the chain. e.g. s2t.json expands 里 to 里 and + // 裏, then t2tw.json passes 里 as-is and converts 裏 to 裡. + if (word_set.insert(converted_word).second) { + converted_words.push_back(converted_word); + } + continue; + } + matched = true; + const opencc::DictEntry* entry = item.Get(); + for (const auto& converted_word : entry->Values()) { + if (word_set.insert(converted_word).second) { + converted_words.push_back(converted_word); + } + } } - return forms->size() > 0; + original_words.swap(converted_words); } + // No dictionary contains the word + if (!matched) return false; + *forms = std::move(original_words); + return forms->size() > 0; } bool Opencc::RandomConvertText(const string& text, string* simplified) { if (dict_ == nullptr) return false; - const char *phrase = text.c_str(); - std::ostringstream buffer; - for (const char* pstr = phrase; *pstr != '\0';) { - opencc::Optional matched = dict_->MatchPrefix(pstr); - size_t matchedLength; - if (matched.IsNull()) { - matchedLength = opencc::UTF8Util::NextCharLength(pstr); - buffer << opencc::UTF8Util::FromSubstr(pstr, matchedLength); - } else { - matchedLength = matched.Get()->KeyLength(); - size_t i = rand() % (matched.Get()->NumValues()); - buffer << matched.Get()->Values().at(i); + const list conversions = + converter_->GetConversionChain()->GetConversions(); + const char* phrase = text.c_str(); + for (auto conversion : conversions) { + opencc::DictPtr dict = conversion->GetDict(); + if (dict == nullptr) return false; + std::ostringstream buffer; + for (const char* pstr = phrase; *pstr != '\0';) { + opencc::Optional matched = + dict->MatchPrefix(pstr); + size_t matched_length; + if (matched.IsNull()) { + matched_length = opencc::UTF8Util::NextCharLength(pstr); + buffer << opencc::UTF8Util::FromSubstr(pstr, matched_length); + } else { + matched_length = matched.Get()->KeyLength(); + size_t i = rand() % (matched.Get()->NumValues()); + buffer << matched.Get()->Values().at(i); + } + pstr += matched_length; } - pstr += matchedLength; + *simplified = buffer.str(); + phrase = simplified->c_str(); } - *simplified = buffer.str(); return *simplified != text; } + // for lua string Opencc::convert_text(const string& text) { string res; From 0f69bfa2f2724c513987855cad71611f8759c703 Mon Sep 17 00:00:00 2001 From: Chunhui He Date: Tue, 27 Feb 2024 12:42:48 +0000 Subject: [PATCH 37/47] fix: build with librime 1.9.0 --- src/modules.cc | 6 ++++-- src/opencc.cc | 5 +++-- src/types.cc | 12 ++++++++---- src/types_ext.cc | 5 ----- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/modules.cc b/src/modules.cc index a4ed5e0..6ca704e 100644 --- a/src/modules.cc +++ b/src/modules.cc @@ -35,11 +35,13 @@ template struct COMPAT().user_data_dir.string())>> { static std::string get_shared_data_dir() { // path::string() returns native encoding on Windows - return rime::Service::instance().deployer().shared_data_dir.string(); + T &deployer = rime::Service::instance().deployer(); + return deployer.shared_data_dir.string(); } static std::string get_user_data_dir() { - return rime::Service::instance().deployer().user_data_dir.string(); + T &deployer = rime::Service::instance().deployer(); + return deployer.user_data_dir.string(); } }; } diff --git a/src/opencc.cc b/src/opencc.cc index 1cf06c9..4413397 100644 --- a/src/opencc.cc +++ b/src/opencc.cc @@ -191,8 +191,9 @@ namespace OpenccReg { template struct COMPAT().user_data_dir.string())>> { static optional make(const string &filename) { - path user_path = Service::instance().deployer().user_data_dir; - path shared_path = Service::instance().deployer().shared_data_dir; + U &deployer = Service::instance().deployer(); + auto user_path = deployer.user_data_dir; + auto shared_path = deployer.shared_data_dir; try{ return T((user_path / "opencc" / filename).u8string()); } diff --git a/src/types.cc b/src/types.cc index 877477d..0f38eb4 100644 --- a/src/types.cc +++ b/src/types.cc @@ -55,19 +55,23 @@ struct COMPAT { template struct COMPAT().user_data_dir.string())>> { static an new_ReverseDb(const std::string &file) { - return New(Service::instance().deployer().user_data_dir / file); + T &deployer = Service::instance().deployer(); + return New(deployer.user_data_dir / file); } static string get_shared_data_dir() { - return Service::instance().deployer().shared_data_dir.string(); + T &deployer = Service::instance().deployer(); + return deployer.shared_data_dir.string(); } static string get_user_data_dir() { - return Service::instance().deployer().user_data_dir.string(); + T &deployer = Service::instance().deployer(); + return deployer.user_data_dir.string(); } static string get_sync_dir() { - return Service::instance().deployer().sync_dir.string(); + T &deployer = Service::instance().deployer(); + return deployer.sync_dir.string(); } }; diff --git a/src/types_ext.cc b/src/types_ext.cc index 99e351d..cd8c6ea 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -275,11 +275,6 @@ namespace UserDbReg{ return {}; } - string file_name(const T& t) { - // returns ANSI encoded string on Windows - return t.file_path().string(); - } - bool Open(T &t) { return t.Open(); } bool Close(T &t) { return t.Close(); } bool OpenReadOnly(T &t) { return t.OpenReadOnly(); } From 740469987dd79098bcb42fd923438c7704a3075a Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Fri, 8 Mar 2024 20:29:07 +0800 Subject: [PATCH 38/47] UserDbReg : add db_pool_ (#320) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UserDbReg : add db_pool_ Signed-off-by: shewer * 改用 script 製作 db_pool_ Signed-off-by: shewer --------- Signed-off-by: shewer --- sample/lua/userdb.lua | 73 +++++++++++++++++++++++++++++++++++++++++++ src/types_ext.cc | 30 ++++++------------ 2 files changed, 82 insertions(+), 21 deletions(-) create mode 100644 sample/lua/userdb.lua diff --git a/sample/lua/userdb.lua b/sample/lua/userdb.lua new file mode 100644 index 0000000..fed86f6 --- /dev/null +++ b/sample/lua/userdb.lua @@ -0,0 +1,73 @@ +#! /usr/bin/env lua +-- +-- userdb.lua +-- Copyright (C) 2024 Shewer Lu +-- +-- Distributed under terms of the MIT license. +-- +--[[ +example: +local userdb = require 'userdb' +local ldb=userdb.LevelDb('ecdict') +ldb:open() +for k,v in ldb:query('a'):iter() do print(k,v) end + +--]] +local db_pool_ = {} +local vars_get= { + _loaded=true, + read_only=true, + disabled=true, + name=true, + file_name=true, + } +local vars_set= {} +local userdb_mt = {} +function userdb_mt.__newindex(tab,key,value) + local db = db_pool_[tab._db_key] + if not db then + db = UserDb(tab._db_name, tab._db_class) + db_pool_[tab._db_key] = db + end + if db and vars_set[key] then + db[key]= value + end +end + +function userdb_mt.__index(tab,key) + local db = db_pool_[tab._db_key] + if not db then + db = UserDb(tab._db_name, tab._db_class) + db_pool_[tab._db_key] = db + end + + if db and vars_get[key] then + return db[key] + else + return function (tab, ...) + return db[key](db,...) + end + end +end + +local userdb= {} + +function userdb.UserDb(db_name, db_class) + local db_key = db_name .. "." .. db_class + local db = { + _db_key = db_key, + _db_name= db_name, + _db_class = db_class, + } + db_pool_[db_key] = UserDb(db_name, db_class) + return setmetatable(db , userdb_mt) +end + +function userdb.LevelDb(db_name) + return userdb.UserDb(db_name, "userdb") +end +function userdb.TableDb(db_name) + return userdb.UserDb(db_name, "plain_userdb") +end + +return userdb diff --git a/src/types_ext.cc b/src/types_ext.cc index cd8c6ea..673fdfa 100644 --- a/src/types_ext.cc +++ b/src/types_ext.cc @@ -252,12 +252,10 @@ namespace UserDbReg{ using A = DbAccessor; an make(const string& db_name, const string& db_class) { - if (auto comp= UserDb::Require(db_class)){ - return an( comp->Create(db_name)) ; - } - else { - return {}; + if (auto comp= Db::Require(db_class)) { + return an(comp->Create(db_name)); } + return {}; } an make_leveldb(const string& db_name) { @@ -275,16 +273,6 @@ namespace UserDbReg{ return {}; } - bool Open(T &t) { return t.Open(); } - bool Close(T &t) { return t.Close(); } - bool OpenReadOnly(T &t) { return t.OpenReadOnly(); } - bool Erase(T &t, const string &key) { return t.Erase(key); } - bool Update(T &t, const string &key, const string &value) { - return t.Update(key, value); - } - - an Query(T &t, const string& key) { return t.Query(key); } - static const luaL_Reg funcs[] = { {"UserDb", WRAP(make)},// an LevelDb( db_file, db_name) {"LevelDb", WRAP(make_leveldb)},// an LevelDb( db_file, db_name) @@ -293,13 +281,13 @@ namespace UserDbReg{ }; static const luaL_Reg methods[] = { - {"open", WRAP(Open)}, - {"open_read_only", WRAP(OpenReadOnly)}, - {"close", WRAP(Close)}, - {"query", WRAP(Query)}, // query(prefix_key) return DbAccessor + {"open", WRAPMEM(T, Open)}, + {"open_read_only", WRAPMEM(T, OpenReadOnly)}, + {"close", WRAPMEM(T, Close)}, + {"query", WRAPMEM(T, Query)}, // query(prefix_key) return DbAccessor {"fetch", WRAP(fetch)}, // fetch(key) return value - {"update", WRAP(Update)}, // update(key,value) return bool - {"erase", WRAP(Erase)}, // erase(key) return bool + {"update", WRAPMEM(T, Update)}, // update(key,value) return bool + {"erase", WRAPMEM(T, Erase)}, // erase(key) return bool {"loaded",WRAPMEM(T, loaded)}, {"disable", WRAPMEM(T, disable)}, From 20ddea907e0b0c9c60d1dcb6b102bee38697cb5c Mon Sep 17 00:00:00 2001 From: fxliang Date: Fri, 8 Mar 2024 20:30:00 +0800 Subject: [PATCH 39/47] KeyEvent() constructor now can also take a code and a modifier (#321) --- src/types.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/types.cc b/src/types.cc index 0f38eb4..5e591e6 100644 --- a/src/types.cc +++ b/src/types.cc @@ -508,12 +508,20 @@ namespace KeyEventReg { return t.modifier(); } - an make(const string &key) { - return New(key) ; + int raw_make(lua_State *L) { + an res; + int n = lua_gettop(L); + if (n == 1) + res = New( string(lua_tostring(L, 1)) ); + else if (n > 1) + res = New( lua_tointeger(L, 1), lua_tointeger(L, 2) ); + + LuaType>::pushdata(L, res); + return 1; } static const luaL_Reg funcs[] = { - { "KeyEvent", WRAP(make) }, + { "KeyEvent", raw_make }, { NULL, NULL }, }; From 7c1b93965962b7c480d4d7f1a947e4712a9f0c5f Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Sat, 13 Apr 2024 11:01:44 +0800 Subject: [PATCH 40/47] ConfigItemReg replace GET macro methods (#323) * ConfigItemReg methods * fixed function name "Get_" Signed-off-by: shewer --------- Signed-off-by: shewer --- src/types.cc | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/src/types.cc b/src/types.cc index 5e591e6..e6c4c3a 100644 --- a/src/types.cc +++ b/src/types.cc @@ -1109,6 +1109,11 @@ namespace ConfigItemReg { using L = ConfigList; using V = ConfigValue; +template +an Get(an t) { + return std::dynamic_pointer_cast(t); +}; + string type(T &t){ switch (t.type()) { case T::kNull: return "kNull"; @@ -1119,30 +1124,37 @@ namespace ConfigItemReg { return ""; } -//START_GET_ -//sed sed -n -e'/\/\/START_GET_/,/\/\/END_GET_/p' src/types.cc | gcc -E - -#define GET_(f_name,from ,rt, k_type) \ - an f_name( an t) { \ - if (t->type() == from::k_type) \ - return std::dynamic_pointer_cast (t);\ - return nullptr;\ + int get_obj(lua_State *L_) { + if (an t = LuaType>::todata(L_, 1)) { + auto t_type = t->type(); + if (T::kScalar == t_type) { + lua_pushcfunction(L_, WRAP(Get)); + } + else if (T::kList == t_type) { + lua_pushcfunction(L_, WRAP(Get)); + } + else if (T::kMap == t_type) { + lua_pushcfunction(L_, WRAP(Get)); + } + else { + return 0; + } + lua_pushvalue(L_, 1); + lua_call(L_, 1, 1); + return 1; + } + return 0; } - GET_( get_value,T, V, kScalar ); - GET_( get_list, T, L, kList ); - GET_( get_map, T, M, kMap ); - -#undef GET_ -//END_GET_ - static const luaL_Reg funcs[] = { { NULL, NULL }, }; static const luaL_Reg methods[] = { - {"get_value",WRAP(get_value)}, - {"get_list",WRAP(get_list)}, - {"get_map",WRAP(get_map)}, + {"get_value",WRAP(Get)}, + {"get_list",WRAP(Get)}, + {"get_map",WRAP(Get)}, + {"get_obj", get_obj}, { NULL, NULL }, }; From c85440f5b11da7aac4e7327f48b8c603669bf392 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Thu, 2 May 2024 11:47:15 +0800 Subject: [PATCH 41/47] add SegmentReg active_text(string) (#329) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 便於取得 substr , 減少 lua string.sub 定位問題 string.sub( inp, seg._start+1 , seg._end ) ```lua function tran.func(inp, seg, env) local active_inp = seg:active_text(inp) .... end ``` --- src/types.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/types.cc b/src/types.cc index e6c4c3a..db316f0 100644 --- a/src/types.cc +++ b/src/types.cc @@ -104,6 +104,10 @@ namespace SegmentReg { t.status = T::kConfirmed; } + string active_text(T &t, const string &r) { + return r.substr(t.start, t.end - t.start); + } + static const luaL_Reg funcs[] = { { "Segment", WRAP(make) }, { NULL, NULL }, @@ -116,6 +120,7 @@ namespace SegmentReg { { "has_tag", WRAPMEM(T::HasTag) }, { "get_candidate_at", WRAPMEM(T::GetCandidateAt) }, { "get_selected_candidate", WRAPMEM(T::GetSelectedCandidate) }, + { "active_text", WRAP(active_text) }, { NULL, NULL }, }; From cb1152c4bb3be910ad26d67470fbc5ff21570cb5 Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Thu, 2 May 2024 11:48:37 +0800 Subject: [PATCH 42/47] refactor set_memorize_callback() memorize_callback (#331) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add Component.TableTranslator Signed-off-by: shewer * add LTableTranslator * 移除 table_translator.h Signed-off-by: shewer * 移除 table_translator.h Signed-off-by: shewer * 加入 LScriptTranslator Signed-off-by: shewer * fixed boost::optional error Signed-off-by: shewer * fix memory dump : memories_callback 初始化 Signed-off-by: shewer * add sample Signed-off-by: shewer * fixd code Signed-off-by: shewer * clang-format 排版 Signed-off-by: shewer * fixed sample/ script code * 補上 *engine Signed-off-by: shewer * add dict user_dict members of Translator Signed-off-by: shewer * refactoring code Signed-off-by: shewer * fixed ScriptTranslatorReg::vars_get: {memorize_call} Signed-off-by: shewer * using optional : memorize_callback() Signed-off-by: shewer * fixed format Signed-off-by: shewer * add raw_set_memorize_callback([nil|function|other data]) return bool: nil(reset), function Signed-off-by: shewer * reduce include head file in translator.h Signed-off-by: shewer * update sample sclipt (table_translator.lua script_translator.lua) Signed-off-by: shewer * add vars_get "lang_name" to support langname check Signed-off-by: shewer --------- Signed-off-by: shewer --- sample/lua/script_translator.lua | 9 ++++++-- sample/lua/table_translator.lua | 9 ++++++-- src/script_translator.cc | 26 ++++++++++++++++-------- src/table_translator.cc | 28 +++++++++++++++++-------- src/translator.h | 35 ++++++++++++++++++++++++-------- 5 files changed, 77 insertions(+), 30 deletions(-) diff --git a/sample/lua/script_translator.lua b/sample/lua/script_translator.lua index fac3955..5dedaa5 100644 --- a/sample/lua/script_translator.lua +++ b/sample/lua/script_translator.lua @@ -21,11 +21,12 @@ env.tran:discard_session false function() env.tran:query false function(inp, seg) env.tran:memorize false function(commit_entrys) env.tran:update_entry false function(entry, state, prefix_str) -env.tran.memorize_callback = function(self, commit_entry) +env.tran:set_memorize_callback bool function(commit_entry) ------- vars_set env.tran.spelling_hints = int >0 env.tran.initial_quality = double env.tran.contextual_suggestions = boolean +env.tran.memorize_callback = [function | nil] env.tran.enable_completion = boolean env.tran.always_show_comments = boolean env.tran.strict_spelling = boolean @@ -38,6 +39,7 @@ env.tran.delimiters = string res = env.tran.spelling_hints 0 number res = env.tran.initial_quality 0.0 number res = env.tran.contextual_suggestions false boolean +env.tran.memorize_callback function|nil res = env.tran.enable_completion true boolean res = env.tran.always_show_comments false boolean res = env.tran.strict_spelling false boolean @@ -64,7 +66,10 @@ local function callback(self, commits) -- self : env.tran commits : list end function M.init(env) env.tran = Component.ScriptTranslator(env.engine, env.name_space, "script_translator") - env.tran:memorize_callback(simple_callback) + env.tran:set_memorize_callback(simple_callback) + --env.tran:set_memorize_callback() -- reset callback + --env.tran.memorize_callback= function(simple_callback) + --env.tran.memorize_callback= nil -- reset callback end function M.fini(env) diff --git a/sample/lua/table_translator.lua b/sample/lua/table_translator.lua index aa92681..fcefa0c 100644 --- a/sample/lua/table_translator.lua +++ b/sample/lua/table_translator.lua @@ -21,12 +21,13 @@ env.tran:discard_session bool function() env.tran:query translation function(inp, seg) env.tran:memorize bool function(commit_entrys) env.tran:update_entry bool function(entry) +env.tran:set_memorize_callback bool function(commit_entry) ------- vars_set 設定值 env.tran.max_homophones = number env.tran.spelling_hints = number env.tran.enable_correction = boolean -env.tran.memorize_callback = function(self, commits) +env.tran.memorize_callback = function(commits)|nil env.tran.enable_completion = false boolean env.tran.delimiters = false string env.tran.strict_spelling = boolean @@ -39,6 +40,7 @@ env.tran.tag = string res = env.tran.max_homophones 1 number res = env.tran.spelling_hints 0 number res = env.tran.enable_correction false boolean +env.tran.memorize_callback function|nil res = env.tran.enable_completion true boolean res = env.tran.delimiters ' string res = env.tran.strict_spelling false boolean @@ -67,7 +69,10 @@ local function callback(self, commits) -- self : env.tran commits : list end function M.init(env) env.tran = Component.TableTranslator(env.engine, env.name_space, "table_translator") - env.tran:memorize_callback(simple_callback) + env.tran:set_memorize_callback(simple_callback) + --env.tran:set_memorize_callback() -- reset callback + --env.tran.memorize_callback= function(simple_callback) + --env.tran.memorize_callback= nil -- reset callback end function M.fini(env) diff --git a/src/script_translator.cc b/src/script_translator.cc index 467f68a..07c4829 100644 --- a/src/script_translator.cc +++ b/src/script_translator.cc @@ -10,6 +10,9 @@ #include #include +#include +#include +#include #include "translator.h" @@ -29,7 +32,8 @@ namespace ScriptTranslatorReg { int commits, const string& new_entory_prefix); SET_(memorize_callback, an); - bool memorize_callback(); + optional> memorize_callback(); + string lang_name() { return language_->name();}; // TranslatorOptions SET_(contextual_suggestions, bool); @@ -53,8 +57,10 @@ namespace ScriptTranslatorReg { using T = LScriptTranslator; - bool T::memorize_callback() { - return (memorize_callback_) ? true : false; + optional> T::memorize_callback() { + if (memorize_callback_) + return memorize_callback_; + return {}; } bool T::memorize(const CommitEntry& commit_entry) { @@ -67,7 +73,7 @@ namespace ScriptTranslatorReg { } auto r = lua_->call, LScriptTranslator*, const CommitEntry&>( - memorize_callback_, this, commit_entry); + memorize_callback_, this, commit_entry); if (!r.ok()) { auto e = r.get_err(); LOG(ERROR) << "LScriptTranslator of " << name_space_ @@ -113,14 +119,15 @@ namespace ScriptTranslatorReg { WMEM(memorize), // delegate TableTransaltor::Momorize WMEM(update_entry), // delegate UserDictionary::UpdateEntry WMEM(reload_user_dict_disabling_patterns), - Set_WMEM(memorize_callback), // an callback function + {"set_memorize_callback", raw_set_memorize_callback}, // an callback function {NULL, NULL}, }; static const luaL_Reg vars_get[] = { - Get_WMEM(name_space), // string - Set_WMEM(memorize_callback), // an callback function - // ScriptTranslator member + Get_WMEM(name_space), // string + Get_WMEM(lang_name), // string + Get_WMEM(memorize_callback), // an callback function + // ScriptTranslator member Get_WMEM(max_homophones), // int Get_WMEM(spelling_hints), // int Get_WMEM(always_show_comments), // bool @@ -141,12 +148,13 @@ namespace ScriptTranslatorReg { }; static const luaL_Reg vars_set[] = { + {"memorize_callback", raw_set_memorize_callback}, // an callback function // ScriptTranslator member Set_WMEM(max_homophones), // int Set_WMEM(spelling_hints), // int Set_WMEM(always_show_comments), // bool Set_WMEM(enable_correction), // bool - // TranslatorOptions + // TranslatorOptions Set_WMEM(delimiters), // string& Set_WMEM(tag), // string Set_WMEM(enable_completion), // bool diff --git a/src/table_translator.cc b/src/table_translator.cc index ef6722b..2b187f9 100644 --- a/src/table_translator.cc +++ b/src/table_translator.cc @@ -11,6 +11,10 @@ #include #include +#include +#include +#include + #include "translator.h" using namespace rime; @@ -29,7 +33,8 @@ namespace TableTranslatorReg { int commits, const string& new_entory_prefix); SET_(memorize_callback, an); - bool memorize_callback(); + optional> memorize_callback(); + string lang_name() const { return language_->name();}; // TranslatorOptions void set_contextual_suggestions(bool); @@ -60,9 +65,12 @@ namespace TableTranslatorReg { using T = LTableTranslator; - bool T::memorize_callback() { - return (memorize_callback_) ? true : false; + optional> T::memorize_callback() { + if (memorize_callback_) + return memorize_callback_; + return {}; } + bool T::memorize(const CommitEntry& commit_entry) { return TableTranslator::Memorize(commit_entry); } @@ -73,18 +81,18 @@ namespace TableTranslatorReg { } auto r = lua_->call, LTableTranslator*, const CommitEntry&>( - memorize_callback_, this, commit_entry); + memorize_callback_, this, commit_entry); if (!r.ok()) { auto e = r.get_err(); LOG(ERROR) << "LTableTranslator of " << name_space_ - << ": memorize_callback error(" << e.status << "): " << e.e; + << ": memorize_callback error(" << e.status << "): " << e.e; return false; } return r.get(); } bool T::update_entry(const DictEntry& entry, - int commits, const string& new_entory_prefix) { + int commits, const string& new_entory_prefix) { if (user_dict_ && user_dict_->loaded()) return user_dict_->UpdateEntry(entry, commits, new_entory_prefix); @@ -151,14 +159,15 @@ namespace TableTranslatorReg { WMEM(memorize), // delegate TableTransaltor::Momorize WMEM(update_entry), // delegate UserDictionary::UpdateEntry WMEM(reload_user_dict_disabling_patterns), - Set_WMEM(memorize_callback), // an callback function + {"set_memorize_callback", raw_set_memorize_callback}, // an callback function {NULL, NULL}, }; static const luaL_Reg vars_get[] = { // class translator member - Get_WMEM(name_space), // string - Get_WMEM(memorize_callback), // an callback function + Get_WMEM(name_space), // string + Get_WMEM(lang_name), // string + Get_WMEM(memorize_callback), // an callback function // TabletTranslator member Get_WMEM(enable_charset_filter), // bool Get_WMEM(enable_encoder), // bool @@ -183,6 +192,7 @@ namespace TableTranslatorReg { }; static const luaL_Reg vars_set[] = { + {"memorize_callback", raw_set_memorize_callback}, // an callback function // TableTranslator member Set_WMEM(enable_charset_filter), // bool Set_WMEM(enable_encoder), // bool diff --git a/src/translator.h b/src/translator.h index d5bd9b2..afcdc4e 100644 --- a/src/translator.h +++ b/src/translator.h @@ -8,13 +8,9 @@ #ifndef _LUA_TRANSLATOR_H #define _LUA_TRANSLATOR_H -#include -#include -#include -#include -#include -#include +#include #include "lib/lua_export_type.h" +#include "optional.h" #define SET_(name, type) \ void set_##name(const type data) { \ @@ -56,9 +52,9 @@ int raw_make_translator(lua_State* L){ if ( n == 4 ) ticket.schema = &(LuaType::todata(L, 2) ); //overwrite schema Lua* lua= Lua::from_state(L); - rime::an obj = rime::New(ticket, lua); + std::shared_ptr obj = rime::New(ticket, lua); if (obj) { - LuaType>::pushdata(L, obj); + LuaType>::pushdata(L, obj); return 1; } else { @@ -66,4 +62,27 @@ int raw_make_translator(lua_State* L){ return 0; } }; + +template +int raw_set_memorize_callback(lua_State *L) { + bool res = false; + auto t = LuaType>::todata(L, 1); + const int type = (1 < lua_gettop(L)) ? lua_type(L, 2) : LUA_TNIL; + if (type == LUA_TNIL) { + // reset memorize_callback + LOG(INFO) << typeid(*t).name() <<" of " << t->name_space() << ": reset memorize_callback"; + t->set_memorize_callback({}); + res = true; + } + else if (type == LUA_TFUNCTION) { + t->set_memorize_callback( LuaObj::todata(L, 2)); + res = true; + } + else { + LOG(WARNING) << typeid(*t).name() <<" of " << t->name_space() + << ": set memorize_callback '?' (function expected, got " << lua_typename(L, type) <<")"; + } + lua_pushboolean(L, res); + return 1; +} #endif /* !_LUA_TRANSLATOR_H */ From 43229d766f1e0f3198f61dc9d2e38bc1f921387f Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Fri, 3 May 2024 21:07:09 +0800 Subject: [PATCH 43/47] add MemoryReg (start_session() finish_session() discard_session() for user_dict update (#333) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在 OnCommit() 線外操作 user_dict 須要 呼叫 start_session() finish_session() --- src/types.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/types.cc b/src/types.cc index db316f0..1450065 100644 --- a/src/types.cc +++ b/src/types.cc @@ -1956,6 +1956,9 @@ namespace MemoryReg { }; static const luaL_Reg methods[] = { + {"start_session", WRAPMEM(T, StartSession)}, + {"finish_session", WRAPMEM(T, FinishSession)}, + {"discard_session", WRAPMEM(T, DiscardSession)}, { "dict_lookup", WRAPMEM(T::dictLookup)}, // bool { "user_lookup", WRAPMEM(T::userLookup)}, // bool { "dictiter_lookup", WRAPMEM(T::dictiterLookup)}, // iter From f5ffb6bec04f935b9a2be52ee82db53283326c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=85=E6=88=8E=E6=B0=8F?= Date: Sun, 19 May 2024 10:35:13 +0800 Subject: [PATCH 44/47] migrate to rime_get_api (#345) the deprecated librime 0.9 API function declarations will be move to a separeate header file rime_api_deprecated.h in https://github.com/rime/librime/pull/877 --- src/types.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.cc b/src/types.cc index 1450065..4976ccc 100644 --- a/src/types.cc +++ b/src/types.cc @@ -36,7 +36,7 @@ template struct COMPAT { // fallback version if librime is old static an new_ReverseDb(const std::string &file) { - return New(std::string(RimeGetUserDataDir()) + "/" + file); + return New(string(rime_get_api()->get_user_data_dir()) + "/" + file); } static string get_shared_data_dir() { From 7be6974b6d81c116bba39f6707dc640f6636fa4e Mon Sep 17 00:00:00 2001 From: Shewer Lu Date: Sun, 19 May 2024 14:30:34 +0800 Subject: [PATCH 45/47] add disconnect to reset dict user_dict (#338) * add ge() in register component Signed-off-by: shewer * add disconnect to reset dict user_dict , remove gc() Signed-off-by: shewer --------- Signed-off-by: shewer --- src/script_translator.cc | 14 +++++++++++++- src/table_translator.cc | 14 +++++++++++++- src/types.cc | 8 +++++++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/script_translator.cc b/src/script_translator.cc index 07c4829..6d9e2d4 100644 --- a/src/script_translator.cc +++ b/src/script_translator.cc @@ -33,7 +33,7 @@ namespace ScriptTranslatorReg { SET_(memorize_callback, an); optional> memorize_callback(); - string lang_name() { return language_->name();}; + string lang_name() const { return (language_) ? language_->name() : "";}; // TranslatorOptions SET_(contextual_suggestions, bool); @@ -49,6 +49,12 @@ namespace ScriptTranslatorReg { GET_(enable_correction, bool); void set_enable_correction(bool); + void disconnect() { + dict_.reset(); + user_dict_.reset(); + language_.reset(); + } + protected: Lua* lua_; an memorize_callback_; @@ -107,6 +113,10 @@ namespace ScriptTranslatorReg { return cl ? user_dict_disabling_patterns_.Load(cl) : false; } + an as_translator(an &t) { + return As(t); + } + static const luaL_Reg funcs[] = { {NULL, NULL}, }; @@ -120,6 +130,7 @@ namespace ScriptTranslatorReg { WMEM(update_entry), // delegate UserDictionary::UpdateEntry WMEM(reload_user_dict_disabling_patterns), {"set_memorize_callback", raw_set_memorize_callback}, // an callback function + {"disconnect", WRAPMEM(T::disconnect)}, {NULL, NULL}, }; @@ -144,6 +155,7 @@ namespace ScriptTranslatorReg { // Memory Get_WMEM(dict), Get_WMEM(user_dict), + {"translator", WRAP(as_translator)}, {NULL, NULL}, }; diff --git a/src/table_translator.cc b/src/table_translator.cc index 2b187f9..155d15c 100644 --- a/src/table_translator.cc +++ b/src/table_translator.cc @@ -34,7 +34,7 @@ namespace TableTranslatorReg { SET_(memorize_callback, an); optional> memorize_callback(); - string lang_name() const { return language_->name();}; + string lang_name() const { return (language_) ? language_->name() : "";}; // TranslatorOptions void set_contextual_suggestions(bool); @@ -55,6 +55,12 @@ namespace TableTranslatorReg { GET_(enable_sentence, bool); void set_enable_sentence(bool); + void disconnect() { + dict_.reset(); + user_dict_.reset(); + language_.reset(); + } + protected: Lua* lua_; an memorize_callback_; @@ -147,6 +153,10 @@ namespace TableTranslatorReg { return cl ? user_dict_disabling_patterns_.Load(cl) : false; } + an as_translator(an &t) { + return As(t); + } + static const luaL_Reg funcs[] = { {NULL, NULL}, }; @@ -160,6 +170,7 @@ namespace TableTranslatorReg { WMEM(update_entry), // delegate UserDictionary::UpdateEntry WMEM(reload_user_dict_disabling_patterns), {"set_memorize_callback", raw_set_memorize_callback}, // an callback function + {"disconnect", WRAPMEM(T::disconnect)}, {NULL, NULL}, }; @@ -188,6 +199,7 @@ namespace TableTranslatorReg { // Memory Get_WMEM(dict), Get_WMEM(user_dict), + {"translator", WRAP(as_translator)}, {NULL, NULL}, }; diff --git a/src/types.cc b/src/types.cc index 4976ccc..7daf538 100644 --- a/src/types.cc +++ b/src/types.cc @@ -1794,7 +1794,7 @@ namespace MemoryReg { const string& new_entry_prefix, const string& lang_string); bool update_candidate(const an cand, const int commits); - const string lang_name() { return language_->name(); }; + const string lang_name() { return (language_) ? language_->name() : ""; }; virtual bool Memorize(const CommitEntry& entry); LuaMemory(const Ticket& ticket, Lua* lua) @@ -1809,6 +1809,11 @@ namespace MemoryReg { void clearUser() { uter.reset(); } + void disconnect() { + dict_.reset(); + user_dict_.reset(); + language_.reset(); + } }; vector LuaMemory::decode(const Code& code) { @@ -1970,6 +1975,7 @@ namespace MemoryReg { { "update_userdict", WRAPMEM(T::update_userdict)}, { "update_entry", WRAPMEM(T::update_entry)}, { "update_candidate", WRAPMEM(T::update_candidate)}, + { "disconnect", WRAPMEM(T::disconnect)}, {NULL, NULL}, }; From 8953d7b71b691250166a39412cb1cd329f96cf82 Mon Sep 17 00:00:00 2001 From: ksqsf Date: Mon, 19 Aug 2024 14:36:30 +0200 Subject: [PATCH 46/47] Wrap Spans (#361) --- src/types.cc | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/types.cc b/src/types.cc index 7daf538..acf91f5 100644 --- a/src/types.cc +++ b/src/types.cc @@ -1991,6 +1991,49 @@ namespace MemoryReg { }; } // namespace MemoryReg +//--- wrappers for Spans +namespace SpansReg { + using T = Spans; + + T make() { + return Spans(); + } + + size_t count(T& spans) { + return spans.Count(); + } + + size_t count_between(T& spans, size_t start, size_t end) { + return spans.Count(start, end); + } + + static const luaL_Reg funcs[] = { + { "Spans", WRAP(make) }, + { NULL, NULL }, + }; + + static const luaL_Reg methods[] = { + { "add_span", WRAPMEM(T, AddSpan) }, + { "add_spans", WRAPMEM(T, AddSpans) }, + { "previous_stop", WRAPMEM(T, PreviousStop) }, + { "next_stop", WRAPMEM(T, NextStop) }, + { "has_vertex", WRAPMEM(T, HasVertex) }, + { "count_between", WRAP(count_between) }, + { NULL, NULL }, + }; + + static const luaL_Reg vars_get[] = { + { "start", WRAPMEM(T, start) }, + { "end", WRAPMEM(T, end) }, + { "count", WRAP(count) }, + { NULL, NULL }, + }; + + static const luaL_Reg vars_set[] = { + { NULL, NULL }, + }; +} // namespace SpansReg + //--- wrappers for Phrase namespace PhraseReg { using T = Phrase; @@ -2019,6 +2062,7 @@ namespace PhraseReg { static const luaL_Reg methods[] = { { "toCandidate", WRAP(toCandidate)}, + { "spans", WRAPMEM(T, spans) }, { NULL, NULL }, }; @@ -2036,7 +2080,6 @@ namespace PhraseReg { { "weight", WRAPMEM(T, weight)}, { "code", WRAPMEM(T, code)}, { "entry", WRAPMEM(T, entry)}, - //span //language doesn't wrap yet, so Wrap it later { NULL, NULL }, }; @@ -2299,6 +2342,7 @@ void types_init(lua_State *L) { EXPORT(DictEntryReg, L); EXPORT(CodeReg, L); EXPORT(CommitEntryReg, L); + EXPORT(SpansReg, L); EXPORT(PhraseReg, L); EXPORT(SentenceReg, L); EXPORT(KeySequenceReg, L); From fa6563cf7b40f3bfbf09e856420bff8de6820558 Mon Sep 17 00:00:00 2001 From: wzy <32936898+Freed-Wu@users.noreply.github.com> Date: Mon, 19 Aug 2024 21:23:22 +0800 Subject: [PATCH 47/47] add LUA_VERSION to allow users to choose lua version (#358) --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 356731e..42f2ebb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,8 @@ +set(LUA_VERSION "lua" CACHE STRING "lua version") if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/lua5.4/lua.h") find_package(PkgConfig) if(PkgConfig_FOUND) - foreach(pkg lua lua54 lua53 lua52 luajit lua51) + foreach(pkg ${LUA_VERSION} lua54 lua53 lua52 luajit lua51) pkg_check_modules(LUA IMPORTED_TARGET GLOBAL ${pkg}) if(LUA_FOUND) break()