diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index edcfd556a..bffdc7fd7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,8 +32,13 @@ jobs: - run: pnpm lint - name: Check CommonJS compatibility run: pnpm compile:cjs + - name: Set NODE_PATH + run: echo "NODE_PATH=$(pwd)/node_modules" >> $GITHUB_ENV - name: Check public API - run: pnpm check:public-api + uses: sap/cloud-sdk-js/.github/actions/check-public-api@main + with: + force_internal_exports: 'false' + ignored_path_pattern: '.*?/client/[^/]+/schema' - name: Check dependencies run: pnpm check:deps - name: License Check diff --git a/package.json b/package.json index d525c42f9..dc699e438 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,6 @@ "schema-tests": "pnpm -F=@sap-ai-sdk/schema-tests", "sample-code": "pnpm -F=@sap-ai-sdk/sample-code", "sample-cap": "pnpm -F=@sap-ai-sdk/sample-cap", - "check:public-api": "pnpm -r check:public-api", "check:deps": "pnpm -r -F !./tests/smoke-tests -F !./tests/schema-tests -F !./sample-cap exec depcheck --ignores=\"nock,@jest/globals\" --quiet" }, "devDependencies": { diff --git a/packages/ai-api/package.json b/packages/ai-api/package.json index 27f3b28d0..8dac68fab 100644 --- a/packages/ai-api/package.json +++ b/packages/ai-api/package.json @@ -25,8 +25,7 @@ "lint": "eslint . && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -c", "lint:fix": "eslint . --fix && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -w --log-level error", "generate": "openapi-generator --generateESM --clearOutputDir -i ./src/spec/AI_CORE_API.yaml -o ./src/client && pnpm update-imports && pnpm lint:fix", - "update-imports": "node --no-warnings --loader ts-node/esm ../../scripts/update-imports.ts ./src/client/AI_CORE_API", - "check:public-api": "node --loader ts-node/esm ../../scripts/check-public-api-cli.ts" + "update-imports": "node --no-warnings --loader ts-node/esm ../../scripts/update-imports.ts ./src/client/AI_CORE_API" }, "dependencies": { "@sap-ai-sdk/core": "workspace:^", diff --git a/packages/core/package.json b/packages/core/package.json index 50d05215d..81cb2ccc3 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -23,8 +23,7 @@ "compile:cjs": "tsc -p tsconfig.cjs.json", "test": "NODE_OPTIONS=--experimental-vm-modules jest", "lint": "eslint \"**/*.ts\" && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -c", - "lint:fix": "eslint \"**/*.ts\" --fix && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -w --log-level error", - "check:public-api": "node --loader ts-node/esm ../../scripts/check-public-api-cli.ts" + "lint:fix": "eslint \"**/*.ts\" --fix && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -w --log-level error" }, "dependencies": { "@sap-cloud-sdk/connectivity": "^3.23.0", diff --git a/packages/foundation-models/package.json b/packages/foundation-models/package.json index ba494eb7e..172a1220f 100644 --- a/packages/foundation-models/package.json +++ b/packages/foundation-models/package.json @@ -25,7 +25,6 @@ "test": "NODE_OPTIONS=--experimental-vm-modules jest", "lint": "eslint \"**/*.ts\" && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -c", "lint:fix": "eslint \"**/*.ts\" --fix && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -w --log-level error", - "check:public-api": "node --loader ts-node/esm ../../scripts/check-public-api-cli.ts", "generate": "pnpm generate:azure-openai", "generate:azure-openai": "openapi-generator --generateESM --clearOutputDir -i ./src/azure-openai/spec/inference.yaml -o ./src/azure-openai/client --schemaPrefix AzureOpenAi", "postgenerate:azure-openai": "rm ./src/azure-openai/client/inference/*.ts && pnpm lint:fix" diff --git a/packages/langchain/package.json b/packages/langchain/package.json index 75342c838..3e95f39b2 100644 --- a/packages/langchain/package.json +++ b/packages/langchain/package.json @@ -23,8 +23,7 @@ "compile:cjs": "tsc -p tsconfig.cjs.json", "test": "NODE_OPTIONS=--experimental-vm-modules jest", "lint": "eslint \"**/*.ts\" && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -c", - "lint:fix": "eslint \"**/*.ts\" --fix && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -w --log-level error", - "check:public-api": "node --loader ts-node/esm ../../scripts/check-public-api-cli.ts" + "lint:fix": "eslint \"**/*.ts\" --fix && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -w --log-level error" }, "dependencies": { "@sap-ai-sdk/ai-api": "workspace:^", diff --git a/packages/orchestration/package.json b/packages/orchestration/package.json index 69b4ce424..24d4b8bb3 100644 --- a/packages/orchestration/package.json +++ b/packages/orchestration/package.json @@ -27,8 +27,7 @@ "lint:fix": "eslint \"**/*.ts\" --fix && prettier . --config ../../.prettierrc --ignore-path ../../.prettierignore -w --log-level error", "generate": "openapi-generator --generateESM --clearOutputDir -i ./src/spec/api.yaml -o ./src/client", "postgenerate": "rm ./src/client/api/*.ts && pnpm rename-generated-files && pnpm lint:fix", - "rename-generated-files": "node --no-warnings --loader ts-node/esm ../../scripts/postgenerate-orchestration.ts ./src/client/api", - "check:public-api": "node --loader ts-node/esm ../../scripts/check-public-api-cli.ts" + "rename-generated-files": "node --no-warnings --loader ts-node/esm ../../scripts/postgenerate-orchestration.ts ./src/client/api" }, "dependencies": { "@sap-ai-sdk/core": "workspace:^", diff --git a/scripts/check-public-api-cli.ts b/scripts/check-public-api-cli.ts deleted file mode 100644 index 1e3bf322b..000000000 --- a/scripts/check-public-api-cli.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { createLogger } from '@sap-cloud-sdk/util'; -import { checkApiOfPackage } from './check-public-api.js'; - -const logger = createLogger('check-public-api'); - -checkApiOfPackage(process.cwd()).catch(err => { - logger.error(err); - process.exit(1); -}); diff --git a/scripts/check-public-api.ts b/scripts/check-public-api.ts deleted file mode 100644 index a266c4e25..000000000 --- a/scripts/check-public-api.ts +++ /dev/null @@ -1,390 +0,0 @@ -/* eslint-disable jsdoc/require-jsdoc */ - -import path, { join, resolve, parse, basename, dirname, posix, sep } from 'path'; -import { fileURLToPath } from 'url'; -import { promises, existsSync } from 'fs'; -import { glob } from 'glob'; -import { createLogger, flatten, unixEOL } from '@sap-cloud-sdk/util'; -import mock from 'mock-fs'; -import { CompilerOptions } from 'typescript'; -import { - readCompilerOptions, - readIncludeExcludeWithDefaults, - transpileDirectory, - defaultPrettierConfig -} from '@sap-cloud-sdk/generator-common/internal.js'; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); - -const { readFile, lstat, readdir } = promises; - -const logger = createLogger('check-public-api'); - -const pathToTsConfigRoot = join(__dirname, '../tsconfig.json'); -const pathRootNodeModules = resolve(__dirname, '../node_modules'); -export const regexExportedIndex = /\{([\w,]+)\}from'\./g; -export const regexExportedInternal = /\.\/([\w-]+)/g; - -function mockFileSystem(pathToPackage: string) { - const { pathToSource, pathToTsConfig, pathToNodeModules, pathToPackageJson } = - paths(pathToPackage); - mock({ - [pathToTsConfig]: mock.load(pathToTsConfig), - [pathToSource]: mock.load(pathToSource), - [pathToPackageJson]: mock.load(pathToPackageJson), - [pathRootNodeModules]: mock.load(pathRootNodeModules), - [pathToNodeModules]: mock.load(pathToNodeModules), - [pathToTsConfigRoot]: mock.load(pathToTsConfigRoot) - }); -} - -function paths(pathToPackage: string): { - pathToSource: string; - pathToPackageJson: string; - pathToTsConfig: string; - pathToNodeModules: string; - pathCompiled: string; -} { - return { - pathToSource: getPathWithPosixSeparator(join(pathToPackage, 'src')), - pathToPackageJson: getPathWithPosixSeparator(join(pathToPackage, 'package.json')), - pathToTsConfig: getPathWithPosixSeparator(join(pathToPackage, 'tsconfig.json')), - pathToNodeModules: getPathWithPosixSeparator(join(pathToPackage, 'node_modules')), - pathCompiled: 'dist' - }; -} - -function getPathWithPosixSeparator(filePath: string): string { - return filePath.split(sep).join(posix.sep); -} - -/** - * Read the compiler options from the root and cwd tsconfig.json. - * @param pathToPackage - Path to the package under investigation. - * @returns The compiler options. - */ -async function getCompilerOptions( - pathToPackage: string -): Promise { - const { pathToSource, pathToTsConfig, pathCompiled } = paths(pathToPackage); - const compilerOptions = await readCompilerOptions(pathToTsConfig); - const compilerOptionsRoot = await readCompilerOptions(pathToTsConfigRoot); - return { - ...compilerOptionsRoot, - ...compilerOptions, - stripInternal: true, - rootDir: pathToSource, - outDir: pathCompiled - }; -} - -/** - * For a detailed explanation what is happening here have a look at `0007-public-api-check.md` in the implementation documentation. - * Here the two sets: exports from index and exports from .d.ts are compared and logs are created. - * @param allExportedIndex - Names of the object imported by the index.ts. - * @param allExportedTypes - Exported object by the .d.ts files. - * @param verbose - Do a lot of detailed output on the packages. - * @returns True if the two sets export the same objects. - */ -function compareApisAndLog( - allExportedIndex: string[], - allExportedTypes: ExportedObject[], - verbose: boolean -): boolean { - let setsAreEqual = true; - const schemaPathRegex = /.*\/client\/[^/]+\/schema\/[^/]+\.ts$/; - - allExportedTypes.forEach(exportedType => { - if ( - !allExportedIndex.find(nameInIndex => exportedType.name === nameInIndex) - ) { - if (getPathWithPosixSeparator(exportedType.path).match(schemaPathRegex)) { - logger.warn( - `The ${exportedType.type} "${exportedType.name}" in file: ${exportedType.path} is not exported in the index.ts.` - ); - return; - } - logger.error( - `The ${exportedType.type} "${exportedType.name}" in file: ${exportedType.path} is neither listed in the index.ts nor marked as internal.` - ); - setsAreEqual = false; - } - }); - - allExportedIndex.forEach(nameInIndex => { - if ( - !allExportedTypes.find(exportedType => exportedType.name === nameInIndex) - ) { - logger.error( - `The object "${nameInIndex}" is exported from the index.ts but marked as @internal.` - ); - setsAreEqual = false; - } - }); - logger.info(`We have found ${allExportedIndex.length} exports.`); - - if (verbose) { - logger.info(`Public api: - ${allExportedIndex.sort().join(`,${unixEOL}`)}`); - } - return setsAreEqual; -} - -/** - * Executes the public API check for a given package. - * @param pathToPackage - Path to the package. - */ -export async function checkApiOfPackage(pathToPackage: string): Promise { - try { - logger.info(`Check package: ${pathToPackage}`); - const { pathToSource, pathCompiled, pathToTsConfig } = paths(pathToPackage); - mockFileSystem(pathToPackage); - const opts = await getCompilerOptions(pathToPackage); - const includeExclude = await readIncludeExcludeWithDefaults(pathToTsConfig); - await transpileDirectory( - pathToSource, - { - compilerOptions: opts, - // We have things in our sources like `#!/usr/bin/env node` in CLI `.js` files which is not working with parser of prettier. - createFileOptions: { - overwrite: true, - prettierOptions: defaultPrettierConfig, - usePrettier: false - } - }, - { exclude: includeExclude?.exclude!, include: ['**/*.ts'] } - ); - // await checkBarrelRecursive(pathToSource); - - const indexFilePath = join(pathToSource, 'index.ts'); - checkIndexFileExists(indexFilePath); - - const allExportedTypes = await parseTypeDefinitionFiles(pathCompiled); - const allExportedIndex = await parseIndexFile(indexFilePath); - - const setsAreEqual = compareApisAndLog( - allExportedIndex, - allExportedTypes, - true - ); - mock.restore(); - if (!setsAreEqual) { - process.exit(1); - } - logger.info( - `The index.ts of package ${pathToPackage} is in sync with the type annotations.` - ); - } finally { - mock.restore(); - } -} - -export function checkIndexFileExists(indexFilePath: string): void { - if (!existsSync(indexFilePath)) { - throw new Error('No index.ts file found in root.'); - } -} - -/** - * Get the paths of all `.d.ts` files. - * @param cwd - Directory which is scanned for type definitions. - * @returns Paths to the `.d.ts` files excluding `index.d.ts` files. - */ -export async function typeDescriptorPaths(cwd: string): Promise { - const files = await glob('**/*.d.ts', { cwd }); - return files - .filter(file => !file.endsWith('index.d.ts')) - .map(file => join(cwd, file)); -} - -export interface ExportedObject { - name: string; - type: string; - path: string; -} - -/** - * Execute the parseTypeDefinitionFile for all files in the cwd. - * @param pathCompiled - Path to the compiled sources containing the .d.ts files. - * @returns Information on the exported objects. - */ -export async function parseTypeDefinitionFiles( - pathCompiled: string -): Promise { - const typeDefinitionPaths = await typeDescriptorPaths(pathCompiled); - const result = await Promise.all( - typeDefinitionPaths.map(async pathTypeDefinition => { - const fileContent = await readFile(pathTypeDefinition, 'utf8'); - const types = parseTypeDefinitionFile(fileContent); - return types.map(type => ({ path: pathTypeDefinition, ...type })); - }) - ); - return flatten(result); -} - -/** - * For a detailed explanation what is happening here have a look at `0007-public-api-check.md` in the implementation documentation. - * Parses a `d.ts` file for the exported objects in it. - * @param fileContent - Content of the .d.ts file to be processes. - * @returns List of exported object. - */ -export function parseTypeDefinitionFile( - fileContent: string -): Omit[] { - const normalized = fileContent.replace(/\n+/g, ''); - return [ - 'function', - 'const', - 'enum', - 'class', - 'abstract class', - 'type', - 'interface' - ].reduce[]>((allObjects, objectType) => { - const regex = - objectType === 'interface' - ? new RegExp(`export ${objectType} (\\w+)`, 'g') - : new RegExp(`export (?:declare )?${objectType} (\\w+)`, 'g'); - const exported = captureGroupsFromGlobalRegex(regex, normalized).map( - element => ({ name: element, type: objectType }) - ); - return [...allObjects, ...exported]; - }, []); -} - -/** - * Parse a barrel file for the exported objects. - * It selects all string in \{\} e.g. export \{a,b,c\} from './xyz' will result in [a,b,c]. - * @param fileContent - Content of the index file to be parsed. - * @param regex - Regular expression used for matching exports. - * @returns List of objects exported by the given index file. - */ -export function parseBarrelFile(fileContent: string, regex: RegExp): string[] { - const normalized = fileContent.replace(/\s+/g, ''); - const groups = captureGroupsFromGlobalRegex(regex, normalized); - - return flatten(groups.map(group => group.split(','))); -} - -// TODO: currently this is called in one place to fix a very specific issue for the AI Core API -// For some reason parsing the barrel file looks at "{ SOMETHING } from '." only, which is not enough and even wrong because it does also look at imports, not only exports -export function parseOtherExports(fileContent: string): string[] { - const normalized = fileContent.replace(/\s+/g, ' '); - const groups = [ - ...captureGroupsFromGlobalRegex(/export const (\w+)/g, normalized), - ...captureGroupsFromGlobalRegex(/export type (\w+)/g, normalized) - ]; - return flatten(groups.map(group => group.split(','))); -} - -function checkInternalReExports(fileContent: string, filePath: string): void { - const internalReExports = parseBarrelFile( - fileContent, - /\{([\w,]+)\}from'.*\/internal'/g - ); - if (internalReExports.length) { - throw new Error( - `Re-exporting internal modules is not allowed. ${internalReExports - .map(reExport => `'${reExport}'`) - .join(', ')} exported in '${filePath}'.` - ); - } -} - -export async function parseIndexFile(filePath: string): Promise { - const cwd = dirname(filePath); - const fileContent = await readFile(filePath, 'utf-8'); - checkInternalReExports(fileContent, filePath); - const localExports = [ - ...parseBarrelFile(fileContent, regexExportedIndex), - ...parseOtherExports(fileContent) - ]; - const starFiles = captureGroupsFromGlobalRegex( - /export \* from '([\w/.-]+)'/g, - fileContent - ); - const starFileExports = await Promise.all( - starFiles.map(async relativeFilePath => { - const fullPath = resolve(cwd, relativeFilePath); - const tsPath = fullPath.replace(/\.js$/, '.ts'); - return parseIndexFile(tsPath); - }) - ); - return [...localExports, ...starFileExports.flat()]; -} - -function captureGroupsFromGlobalRegex(regex: RegExp, str: string): string[] { - const groups = Array.from(str.matchAll(regex)); - return groups.map(group => group[1]); -} - -export async function checkBarrelRecursive(cwd: string): Promise { - (await readdir(cwd, { withFileTypes: true })) - .filter(dirent => dirent.isDirectory()) - .forEach(async subDir => { - if (!['__snapshot__', 'spec', 'tests'].includes(subDir.name)) { - await checkBarrelRecursive(join(cwd, subDir.name)); - } - }); - await exportAllInBarrel( - cwd, - parse(cwd).name === 'src' ? 'internal.ts' : 'index.ts' - ); -} - -export async function exportAllInBarrel( - cwd: string, - barrelFileName: string -): Promise { - const barrelFilePath = join(cwd, barrelFileName); - if (existsSync(barrelFilePath) && (await lstat(barrelFilePath)).isFile()) { - const dirContents = ( - await glob('*', { - ignore: [ - '**/*.test.ts', - '__snapshots__', - 'spec', - 'tests', - 'internal.ts', - 'index.ts', - 'cli.ts', - '**/*.md' - ], - cwd - }) - ).map(name => basename(name, '.ts')); - const exportedFiles = parseBarrelFile( - await readFile(barrelFilePath, 'utf8'), - regexExportedInternal - ); - if (compareBarrels(dirContents, exportedFiles, barrelFilePath)) { - throw Error(`'${barrelFileName}' is not in sync.`); - } - } else { - throw Error(`No '${barrelFileName}' file found in '${cwd}'.`); - } -} - -function compareBarrels( - dirContents: string[], - exportedFiles: string[], - barrelFilePath: string -) { - const missingBarrelExports = dirContents.filter( - x => !exportedFiles.includes(x) - ); - missingBarrelExports.forEach(tsFiles => - logger.error(`'${tsFiles}' is not exported in '${barrelFilePath}'.`) - ); - - const extraBarrelExports = exportedFiles.filter( - x => !dirContents.includes(x) - ); - extraBarrelExports.forEach(exports => - logger.error( - `'${exports}' is exported from the '${barrelFilePath}' but does not exist in this directory.` - ) - ); - - return missingBarrelExports.length || extraBarrelExports.length; -}