feat: Provide typed model parameters in LLM Module Config (#424)

* make model_params optional * changeset * add convenience function * public api * fix: Changes from lint * type def and update readme * remove comment * remove all cases of empty model_params * fix type test * conflict fix * changelog text * remove empty model_param from Readme --------- Co-authored-by: cloud-sdk-js <[email protected]> Co-authored-by: Deeksha Sinha <[email protected]>
SAP · Jan 10, 2025 · 1476584 · 1476584
1 parent 1da2caa
commit 1476584
Show file tree

Hide file tree

Showing 14 changed files with 58 additions and 45 deletions.
diff --git a/.changeset/sharp-pianos-press.md b/.changeset/sharp-pianos-press.md
@@ -0,0 +1,5 @@
+---
+'@sap-ai-sdk/orchestration': minor
+---
+
+[Improvement] Refine the type definition of the `model_params` property in the `LlmModuleConfig` to also include known properties.
diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md
@@ -64,9 +64,10 @@ Consequently, each orchestration deployment uniquely maps to a resource group wi
 ## Usage
 
 Leverage the orchestration service capabilities by using the orchestration client.
-Configure the LLM module by setting the `model_name` and `model_params` properties.
+Configure the LLM module by setting the `model_name` property.
 Define the optional `model_version` property to choose an available model version.
 By default, the version is set to `latest`.
+Specify the optional `model_params` property to apply specific parameters to the model
 
 ```ts
 import { OrchestrationClient } from '@sap-ai-sdk/orchestration';
@@ -203,8 +204,7 @@ import { OrchestrationClient } from '@sap-ai-sdk/orchestration';
 
 const orchestrationClient = new OrchestrationClient({
   llm: {
-    model_name: 'gpt-4o',
-    model_params: {}
+    model_name: 'gpt-4o'
   },
   templating: {
     template: [
@@ -299,8 +299,7 @@ You can anonymize or pseudonomize the prompt using the data masking capabilities
 ```ts
 const orchestrationClient = new OrchestrationClient({
   llm: {
-    model_name: 'gpt-4o',
-    model_params: {}
+    model_name: 'gpt-4o'
   },
   templating: {
     template: [
@@ -335,8 +334,7 @@ Grounding enables integrating external, contextually relevant, domain-specific,
 ```ts
 const orchestrationClient = new OrchestrationClient({
   llm: {
-    model_name: 'gpt-35-turbo',
-    model_params: {}
+    model_name: 'gpt-35-turbo'
   },
   templating: {
     template: [

diff --git a/packages/orchestration/src/client/api/schema/llm-module-config.ts b/packages/orchestration/src/client/api/schema/llm-module-config.ts
@@ -23,7 +23,7 @@ export type LlmModuleConfig = {
    *   "n": 2
    * }
    */
-  model_params: Record<string, any>;
+  model_params?: Record<string, any>;
   /**
    * Version of the model to use
    * Default: "latest".

diff --git a/packages/orchestration/src/index.ts b/packages/orchestration/src/index.ts
@@ -41,12 +41,13 @@ export type {
 export type {
   OrchestrationModuleConfig,
   LlmModuleConfig,
-  Prompt
+  Prompt,
+  LlmModelParams
 } from './orchestration-types.js';
 
 export { OrchestrationClient } from './orchestration-client.js';
 
-export { buildAzureContentFilter } from './orchestration-filter-utility.js';
+export { buildAzureContentFilter } from './orchestration-utils.js';
 
 export { OrchestrationResponse } from './orchestration-response.js';
 

diff --git a/packages/orchestration/src/internal.ts b/packages/orchestration/src/internal.ts
@@ -1,4 +1,4 @@
 export * from './orchestration-client.js';
-export * from './orchestration-filter-utility.js';
+export * from './orchestration-utils.js';
 export * from './orchestration-types.js';
 export * from './orchestration-response.js';
diff --git a/packages/orchestration/src/orchestration-client.test.ts b/packages/orchestration/src/orchestration-client.test.ts
@@ -10,7 +10,7 @@ import {
   constructCompletionPostRequest,
   OrchestrationClient
 } from './orchestration-client.js';
-import { buildAzureContentFilter } from './orchestration-filter-utility.js';
+import { buildAzureContentFilter } from './orchestration-utils.js';
 import { OrchestrationResponse } from './orchestration-response.js';
 import type { CompletionPostResponse } from './client/api/schema/index.js';
 import type {
@@ -268,8 +268,7 @@ describe('orchestration service client', () => {
   it('calls chatCompletion with grounding configuration', async () => {
     const config: OrchestrationModuleConfig = {
       llm: {
-        model_name: 'gpt-35-turbo',
-        model_params: {}
+        model_name: 'gpt-35-turbo'
       },
       templating: {
         template: [
@@ -337,8 +336,7 @@ describe('orchestration service client', () => {
 
     const config: OrchestrationModuleConfig = {
       llm: {
-        model_name: 'gpt-4o',
-        model_params: {}
+        model_name: 'gpt-4o'
       },
       templating: {
         template: [{ role: 'user', content: "What's my name?" }]

diff --git a/packages/orchestration/src/orchestration-completion-post-request.test.ts b/packages/orchestration/src/orchestration-completion-post-request.test.ts
@@ -1,5 +1,5 @@
 import { constructCompletionPostRequest } from './orchestration-client.js';
-import { buildAzureContentFilter } from './orchestration-filter-utility.js';
+import { buildAzureContentFilter } from './orchestration-utils.js';
 import type { CompletionPostRequest } from './client/api/schema';
 import type { OrchestrationModuleConfig } from './orchestration-types.js';
 

diff --git a/packages/orchestration/src/orchestration-types.ts b/packages/orchestration/src/orchestration-types.ts
@@ -29,8 +29,21 @@ export interface Prompt {
 export type LlmModuleConfig = OriginalLlmModuleConfig & {
   /** */
   model_name: ChatModel;
+  model_params?: LlmModelParams;
 };
 
+/**
+ * Model Parameters for LLM module configuration.
+ */
+export type LlmModelParams = {
+  max_tokens?: number;
+  temperature?: number;
+  frequency_penalty?: number;
+  presence_penalty?: number;
+  top_p?: number;
+  n?: number;
+} & Record<string, any>;
+
 /**
  * Orchestration module configuration.
  */

diff --git a/.../src/orchestration-filter-utility.test.ts → ...estration/src/orchestration-utils.test.ts b/.../src/orchestration-filter-utility.test.ts → ...estration/src/orchestration-utils.test.ts
@@ -1,9 +1,9 @@
 import { constructCompletionPostRequest } from './orchestration-client.js';
-import { buildAzureContentFilter } from './orchestration-filter-utility.js';
+import { buildAzureContentFilter } from './orchestration-utils.js';
 import type {
   CompletionPostRequest,
   FilteringModuleConfig
-} from './client/api/schema';
+} from './client/api/schema/index.js';
 import type { OrchestrationModuleConfig } from './orchestration-types.js';
 
 describe('filter utility', () => {

diff --git a/...ation/src/orchestration-filter-utility.ts → .../orchestration/src/orchestration-utils.ts b/...ation/src/orchestration-filter-utility.ts → .../orchestration/src/orchestration-utils.ts
diff --git a/packages/orchestration/src/spec/api.yaml b/packages/orchestration/src/spec/api.yaml
@@ -270,7 +270,6 @@ components:
       type: object
       required:
         - model_name
-        - model_params
       additionalProperties: false
       properties:
         model_name:

diff --git a/sample-cap/srv/orchestration/orchestration-service.ts b/sample-cap/srv/orchestration/orchestration-service.ts
@@ -4,8 +4,7 @@ export default class OrchestrationService {
   async chatCompletion(req: any) {
     const { template, inputParams } = req.data;
     const llm = {
-      model_name: 'gpt-4-32k',
-      model_params: {}
+      model_name: 'gpt-4-32k'
     };
     const templating = { template };
 

diff --git a/sample-code/src/orchestration.ts b/sample-code/src/orchestration.ts
@@ -22,8 +22,7 @@ export async function orchestrationChatCompletion(): Promise<OrchestrationRespon
   const orchestrationClient = new OrchestrationClient({
     // define the language model to be used
     llm: {
-      model_name: 'gpt-4o',
-      model_params: {}
+      model_name: 'gpt-4o'
     },
     // define the prompt
     templating: {
@@ -41,8 +40,7 @@ export async function orchestrationChatCompletion(): Promise<OrchestrationRespon
 }
 
 const llm: LlmModuleConfig = {
-  model_name: 'gpt-4o',
-  model_params: {}
+  model_name: 'gpt-4o'
 };
 
 /**
@@ -160,8 +158,7 @@ export async function orchestrationCompletionMasking(): Promise<
 > {
   const orchestrationClient = new OrchestrationClient({
     llm: {
-      model_name: 'gpt-4-32k',
-      model_params: {}
+      model_name: 'gpt-4-32k'
     },
     templating: {
       template: [

diff --git a/tests/type-tests/test/orchestration.test-d.ts b/tests/type-tests/test/orchestration.test-d.ts
@@ -1,10 +1,11 @@
-import { expectError, expectType } from 'tsd';
+import { expectError, expectType, expectAssignable } from 'tsd';
 import {
   OrchestrationClient,
   CompletionPostResponse,
   OrchestrationResponse,
   TokenUsage,
-  ChatModel
+  ChatModel,
+  LlmModelParams
 } from '@sap-ai-sdk/orchestration';
 
 /**
@@ -16,8 +17,7 @@ expectType<Promise<OrchestrationResponse>>(
       template: [{ role: 'user', content: 'Hello!' }]
     },
     llm: {
-      model_name: 'gpt-35-turbo-16k',
-      model_params: {}
+      model_name: 'gpt-35-turbo-16k'
     }
   }).chatCompletion()
 );
@@ -29,8 +29,7 @@ expectType<CompletionPostResponse>(
         template: [{ role: 'user', content: 'Hello!' }]
       },
       llm: {
-        model_name: 'gpt-35-turbo-16k',
-        model_params: {}
+        model_name: 'gpt-35-turbo-16k'
       }
     }).chatCompletion()
   ).data
@@ -43,8 +42,7 @@ expectType<string | undefined>(
         template: [{ role: 'user', content: 'Hello!' }]
       },
       llm: {
-        model_name: 'gpt-35-turbo-16k',
-        model_params: {}
+        model_name: 'gpt-35-turbo-16k'
       }
     }).chatCompletion()
   ).getContent()
@@ -57,8 +55,7 @@ expectType<string | undefined>(
         template: [{ role: 'user', content: 'Hello!' }]
       },
       llm: {
-        model_name: 'gpt-35-turbo-16k',
-        model_params: {}
+        model_name: 'gpt-35-turbo-16k'
       }
     }).chatCompletion()
   ).getFinishReason()
@@ -71,8 +68,7 @@ expectType<TokenUsage>(
         template: [{ role: 'user', content: 'Hello!' }]
       },
       llm: {
-        model_name: 'gpt-35-turbo-16k',
-        model_params: {}
+        model_name: 'gpt-35-turbo-16k'
       }
     }).chatCompletion()
   ).getTokenUsage()
@@ -85,8 +81,7 @@ expectType<Promise<OrchestrationResponse>>(
         template: [{ role: 'user', content: 'Hello!' }]
       },
       llm: {
-        model_name: 'gpt-35-turbo-16k',
-        model_params: {}
+        model_name: 'gpt-35-turbo-16k'
       }
     },
     {
@@ -197,8 +192,7 @@ expectError<any>(new OrchestrationClient({}).chatCompletion());
 expectError<any>(
   new OrchestrationClient({
     llm: {
-      model_name: 'gpt-35-turbo-16k',
-      model_params: {}
+      model_name: 'gpt-35-turbo-16k'
     }
   }).chatCompletion()
 );
@@ -212,11 +206,20 @@ expectError<any>(
       template: [{ role: 'user', content: 'Hello!' }]
     },
     llm: {
-      model_params: {}
+      model_params: { max_tokens: 50 }
     }
   }).chatCompletion()
 );
 
+/**
+ * Model parameters should accept known typed parameters and arbitrary parameters.
+ */
+expectAssignable<LlmModelParams>({
+  max_tokens: 50,
+  temperature: 0.2,
+  random_property: 'random - value'
+});
+
 /**
  * Model parameters should adhere to OrchestrationCompletionParameters.// Todo: Check if additional checks can be added for model_params.
  */