diff --git a/js/src/core.ts b/js/src/core.ts index d6341526..63e261ef 100644 --- a/js/src/core.ts +++ b/js/src/core.ts @@ -267,6 +267,8 @@ export function getEncodingNameForModel(model: TiktokenModel) { case "gpt-4-1106-preview": case "gpt-4-vision-preview": case "gpt-3.5-turbo-0125": + case "gpt-4-turbo": + case "gpt-4-turbo-2024-04-09": case "gpt-4-turbo-preview": case "gpt-4-0125-preview": case "text-embedding-ada-002": { diff --git a/tiktoken/model_to_encoding.json b/tiktoken/model_to_encoding.json index 1001fec8..f527ffbf 100644 --- a/tiktoken/model_to_encoding.json +++ b/tiktoken/model_to_encoding.json @@ -47,6 +47,8 @@ "gpt-4-32k": "cl100k_base", "gpt-4-32k-0314": "cl100k_base", "gpt-4-32k-0613": "cl100k_base", + "gpt-4-turbo": "cl100k_base", + "gpt-4-turbo-2024-04-09": "cl100k_base", "gpt-4-turbo-preview": "cl100k_base", "gpt-4-1106-preview": "cl100k_base", "gpt-4-0125-preview": "cl100k_base", diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 80f005c8..85630909 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -398,6 +398,8 @@ export type TiktokenModel = | "gpt-4-32k" | "gpt-4-32k-0314" | "gpt-4-32k-0613" + | "gpt-4-turbo" + | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-1106-preview" | "gpt-4-0125-preview" @@ -468,6 +470,8 @@ pub fn encoding_for_model( "gpt-4-1106-preview" => Ok("cl100k_base"), "gpt-4-vision-preview" => Ok("cl100k_base"), "gpt-3.5-turbo-0125" => Ok("cl100k_base"), + "gpt-4-turbo" => Ok("cl100k_base"), + "gpt-4-turbo-2024-04-09" => Ok("cl100k_base"), "gpt-4-turbo-preview" => Ok("cl100k_base"), "gpt-4-0125-preview" => Ok("cl100k_base"), model => Err(JsError::new(