Merge pull request #97 from akynau/add-gpt-4-turbo

Add GPT-4 Turbo
dqbd · Apr 12, 2024 · 1d8a762 · 1d8a762
2 parents 110eef4 + 3768231
commit 1d8a762
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 0 deletions.
diff --git a/js/src/core.ts b/js/src/core.ts
@@ -267,6 +267,8 @@ export function getEncodingNameForModel(model: TiktokenModel) {
     case "gpt-4-1106-preview":
     case "gpt-4-vision-preview":
     case "gpt-3.5-turbo-0125":
+    case "gpt-4-turbo":
+    case "gpt-4-turbo-2024-04-09":
     case "gpt-4-turbo-preview":
     case "gpt-4-0125-preview":
     case "text-embedding-ada-002": {

diff --git a/tiktoken/model_to_encoding.json b/tiktoken/model_to_encoding.json
@@ -47,6 +47,8 @@
     "gpt-4-32k": "cl100k_base",
     "gpt-4-32k-0314": "cl100k_base",
     "gpt-4-32k-0613": "cl100k_base",
+    "gpt-4-turbo": "cl100k_base",
+    "gpt-4-turbo-2024-04-09": "cl100k_base",
     "gpt-4-turbo-preview": "cl100k_base",
     "gpt-4-1106-preview": "cl100k_base",
     "gpt-4-0125-preview": "cl100k_base",

diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs
@@ -398,6 +398,8 @@ export type TiktokenModel =
     | "gpt-4-32k"
     | "gpt-4-32k-0314"
     | "gpt-4-32k-0613"
+    | "gpt-4-turbo"
+    | "gpt-4-turbo-2024-04-09"
     | "gpt-4-turbo-preview"
     | "gpt-4-1106-preview"
     | "gpt-4-0125-preview"
@@ -468,6 +470,8 @@ pub fn encoding_for_model(
         "gpt-4-1106-preview" => Ok("cl100k_base"),
         "gpt-4-vision-preview" => Ok("cl100k_base"),
         "gpt-3.5-turbo-0125" => Ok("cl100k_base"),
+        "gpt-4-turbo" => Ok("cl100k_base"),
+        "gpt-4-turbo-2024-04-09" => Ok("cl100k_base"),
         "gpt-4-turbo-preview" => Ok("cl100k_base"),
         "gpt-4-0125-preview" => Ok("cl100k_base"),
         model => Err(JsError::new(