Add Tokenizer Helper class to ERNIE-Bot.SDK

src/ERNIE-Bot.SDK/Tokenizer.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace ERNIE_Bot.SDK
+{
+    /// <summary>
+    /// This class provides methods for tokenizing text.
+    /// </summary>
+    public static class Tokenizer
+    {
+        public static int ApproxNumTokens(string text)
+        {
+            int chinese = Regex.Matches(text, @"\p{IsCJKUnifiedIdeographs}").Count;
+            int english = Regex.Replace(text, @"[^\p{IsBasicLatin}-]", " ")
+                .Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)
+                .Count(w => !string.IsNullOrWhiteSpace(w) && w != "-" && w != "_");
+
+            return chinese + (int)Math.Floor(english * 1.3);
+        }
+    }
+}

tests/ERNIE-Bot.SDK.Tests/TokenizerTests.cs

@@ -0,0 +1,16 @@
+using ERNIE_Bot.SDK;
+
+namespace ERNIE_Bot.SDK.Tests
+{
+    public class TokenizerTests
+    {
+        [Fact]
+        public void TestApproxNumTokens()
+        {
+            string text = "这是一段测试文字This is a test string.";
+            int expected = 14;
+            int actual = Tokenizer.ApproxNumTokens(text);
+            Assert.Equal(expected, actual);
+        }
+    }
+}