From 0c7e2d439f974d64669f73646403cca99f725537 Mon Sep 17 00:00:00 2001 From: cu <42832096+cuhland@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:56:02 +0100 Subject: [PATCH] feat: Add license text as evidence (#1243) fixes #256 This PR is based on #427, and uses the same implementation as it was implemented in yarn (See https://github.com/CycloneDX/cyclonedx-node-yarn/pull/193) closes #427 --------- Signed-off-by: Matthias Schiebel Signed-off-by: Christoph Uhland <42832096+cuhland@users.noreply.github.com> Signed-off-by: Christoph Uhland Co-authored-by: Matthias Schiebel --- README.md | 2 + src/_helpers.ts | 23 +++++++++ src/builders.ts | 59 ++++++++++++++++++++-- src/cli.ts | 10 +++- tests/unit/_helpers.spec.js | 56 +++++++++++++++++++++ tests/unit/builders.BomBuilder.spec.js | 69 ++++++++++++++++++++++++++ 6 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 tests/unit/_helpers.spec.js create mode 100644 tests/unit/builders.BomBuilder.spec.js diff --git a/README.md b/README.md index b92eeb5e7..22439d278 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,8 @@ Options: --output-reproducible Whether to go the extra mile and make the output reproducible. This requires more resources, and might result in loss of time- and random-based-values. (env: BOM_REPRODUCIBLE) + --gather-license-texts Search for license files in components and include them as license evidence. + This feature is experimental. (default: false) --output-format Which output format to use. (choices: "JSON", "XML", default: "JSON") --output-file Path to the output file. diff --git a/src/_helpers.ts b/src/_helpers.ts index 082ded590..116fbb525 100644 --- a/src/_helpers.ts +++ b/src/_helpers.ts @@ -18,6 +18,7 @@ Copyright (c) OWASP Foundation. All Rights Reserved. */ import { readFileSync, writeSync } from 'fs' +import { parse } from 'path' export function loadJsonFile (path: string): any { return JSON.parse(readFileSync(path, 'utf8')) @@ -56,3 +57,25 @@ export function tryRemoveSecretsFromUrl (url: string): string { return url } } + +export const LICENSE_FILENAME_PATTERN = /^(?:UN)?LICEN[CS]E|.\.LICEN[CS]E$|^NOTICE$/i +const LICENSE_FILENAME_BASE = new Set(['licence', 'license']) +const LICENSE_FILENAME_EXT = new Set(['.apache', '.bsd', '.gpl', '.mit']) +const MAP_TEXT_EXTENSION_MIME = new Map([ + ['', 'text/plain'], + ['.htm', 'text/html'], + ['.html', 'text/html'], + ['.md', 'text/markdown'], + ['.txt', 'text/plain'], + ['.rst', 'text/prs.fallenstein.rst'], + ['.xml', 'text/xml'], + ['.license', 'text/plain'], + ['.licence', 'text/plain'] +]) + +export function getMimeForLicenseFile (filename: string): string | undefined { + const { name, ext } = parse(filename.toLowerCase()) + return LICENSE_FILENAME_BASE.has(name) && LICENSE_FILENAME_EXT.has(ext) + ? 'text/plain' + : MAP_TEXT_EXTENSION_MIME.get(ext) +} diff --git a/src/builders.ts b/src/builders.ts index 24d399f76..cf7a9b541 100644 --- a/src/builders.ts +++ b/src/builders.ts @@ -18,11 +18,17 @@ Copyright (c) OWASP Foundation. All Rights Reserved. */ import { type Builders, Enums, type Factories, Models, Utils } from '@cyclonedx/cyclonedx-library' -import { existsSync } from 'fs' +import { existsSync, readdirSync, readFileSync } from 'fs' import * as normalizePackageData from 'normalize-package-data' import * as path from 'path' - -import { isString, loadJsonFile, tryRemoveSecretsFromUrl } from './_helpers' +import { join } from 'path' + +import { + getMimeForLicenseFile, + isString, + LICENSE_FILENAME_PATTERN, + loadJsonFile, tryRemoveSecretsFromUrl +} from './_helpers' import { makeNpmRunner, type runFunc } from './npmRunner' import { PropertyNames, PropertyValueBool } from './properties' import { versionCompare } from './versionCompare' @@ -37,6 +43,7 @@ interface BomBuilderOptions { reproducible?: BomBuilder['reproducible'] flattenComponents?: BomBuilder['flattenComponents'] shortPURLs?: BomBuilder['shortPURLs'] + gatherLicenseTexts?: BomBuilder['gatherLicenseTexts'] } type cPath = string @@ -56,6 +63,7 @@ export class BomBuilder { reproducible: boolean flattenComponents: boolean shortPURLs: boolean + gatherLicenseTexts: boolean console: Console @@ -79,6 +87,7 @@ export class BomBuilder { this.reproducible = options.reproducible ?? false this.flattenComponents = options.flattenComponents ?? false this.shortPURLs = options.shortPURLs ?? false + this.gatherLicenseTexts = options.gatherLicenseTexts ?? false this.console = console_ } @@ -465,6 +474,23 @@ export class BomBuilder { l.acknowledgement = Enums.LicenseAcknowledgement.Declared }) + if (this.gatherLicenseTexts) { + if (this.packageLockOnly) { + this.console.warn('WARN | Adding license text is ignored (package-lock-only is configured!) for %j', data.name) + } else { + component.evidence = new Models.ComponentEvidence() + for (const license of this.fetchLicenseEvidence(data?.path as string)) { + if (license != null) { + // only create a evidence if a license attachment is found + if (component.evidence == null) { + component.evidence = new Models.ComponentEvidence() + } + component.evidence.licenses.add(license) + } + } + } + } + if (isOptional || isDevOptional) { component.scope = Enums.ComponentScope.Optional } @@ -610,6 +636,33 @@ export class BomBuilder { } } } + + private * fetchLicenseEvidence (path: string): Generator { + const files = readdirSync(path) + for (const file of files) { + if (!LICENSE_FILENAME_PATTERN.test(file)) { + continue + } + + const contentType = getMimeForLicenseFile(file) + if (contentType === undefined) { + continue + } + + const fp = join(path, file) + yield new Models.NamedLicense( + `file: ${file}`, + { + text: new Models.Attachment( + readFileSync(fp).toString('base64'), + { + contentType, + encoding: Enums.AttachmentEncoding.Base64 + } + ) + }) + } + } } class DummyComponent extends Models.Component { diff --git a/src/cli.ts b/src/cli.ts index 58165f068..402363ba8 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -47,6 +47,7 @@ interface CommandOptions { flattenComponents: boolean shortPURLs: boolean outputReproducible: boolean + gatherLicenseTexts: boolean outputFormat: OutputFormat outputFile: string validate: boolean | undefined @@ -115,6 +116,12 @@ function makeCommand (process: NodeJS.Process): Command { ).env( 'BOM_REPRODUCIBLE' ) + ).addOption( + new Option( + '--gather-license-texts', + 'Search for license files in components and include them as license evidence.\n' + + 'This feature is experimental. (default: false)' + ).default(false) ).addOption( (function () { const o = new Option( @@ -249,7 +256,8 @@ export async function run (process: NodeJS.Process): Promise { omitDependencyTypes: options.omit, reproducible: options.outputReproducible, flattenComponents: options.flattenComponents, - shortPURLs: options.shortPURLs + shortPURLs: options.shortPURLs, + gatherLicenseTexts: options.gatherLicenseTexts }, myConsole ).buildFromProjectDir(projectDir, process) diff --git a/tests/unit/_helpers.spec.js b/tests/unit/_helpers.spec.js new file mode 100644 index 000000000..d1d1b2dad --- /dev/null +++ b/tests/unit/_helpers.spec.js @@ -0,0 +1,56 @@ +/*! +This file is part of CycloneDX generator for NPM projects. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +Copyright (c) OWASP Foundation. All Rights Reserved. +*/ + +const { getMimeForLicenseFile, LICENSE_FILENAME_PATTERN } = require('../../dist/_helpers') +const { describe, expect, test } = require('@jest/globals') + +describe('LICENSE_FILENAME_PATTERN', () => { + test.each([ + 'LICENCE', + 'licence', + 'LICENSE', + 'license', + 'NOTICE', + 'UNLICENCE', + 'UNLICENSE'])('valid name: %s', (fileName) => { + const value = LICENSE_FILENAME_PATTERN.test(fileName) + expect(value).toBeTruthy() + }) + test.each([ + 'my-license', + 'my_license', + 'myNotice', + 'the-LICENSE'])('invalid name: %s', (fileName) => { + const value = LICENSE_FILENAME_PATTERN.test(fileName) + expect(value).toBeFalsy() + }) +}) + +describe('getMimeForLicenseFile', () => { + test.each([ + ['LICENCE', 'text/plain'], + ['site.html', 'text/html'], + ['license.md', 'text/markdown'], + ['info.xml', 'text/xml'], + ['UNKNOWN', 'text/plain'] + ])('check %s', (fileName, expected) => { + const value = getMimeForLicenseFile(fileName) + expect(value).toBe(expected) + }) +}) diff --git a/tests/unit/builders.BomBuilder.spec.js b/tests/unit/builders.BomBuilder.spec.js new file mode 100644 index 000000000..e59093a4c --- /dev/null +++ b/tests/unit/builders.BomBuilder.spec.js @@ -0,0 +1,69 @@ +/*! +This file is part of CycloneDX generator for NPM projects. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +Copyright (c) OWASP Foundation. All Rights Reserved. +*/ + +const { Models, Builders, Factories } = require('@cyclonedx/cyclonedx-library') +const fs = require('fs') +const { BomBuilder, TreeBuilder } = require('../../dist/builders') +const { jest, describe, expect, it } = require('@jest/globals') + +describe('BomBuilder', () => { + const extRefFactory = new Factories.FromNodePackageJson.ExternalReferenceFactory() + const bomBuilder = new BomBuilder( + new Builders.FromNodePackageJson.ToolBuilder(extRefFactory), + new Builders.FromNodePackageJson.ComponentBuilder( + extRefFactory, + new Factories.LicenseFactory() + ), + new TreeBuilder(), + new Factories.FromNodePackageJson.PackageUrlFactory('npm'), + { + ignoreNpmErrors: true, + metaComponentType: true, + packageLockOnly: true, + omitDependencyTypes: [], + reproducible: true, + flattenComponents: true, + shortPURLs: true, + gatherLicenseTexts: true + }, + null + ) + describe('License fetching', () => { + it('fetches existing license from directory', async () => { + const fsMock = jest.spyOn(fs, 'readdirSync') + const fileMock = jest.spyOn(fs, 'readFileSync') + fsMock.mockReturnValue(['license.txt']) + fileMock.mockReturnValue('license file content') + const licenses = bomBuilder.fetchLicenseEvidence('test_module') + const license = licenses.next().value + expect(license).toBeInstanceOf(Models.NamedLicense) + expect(license.name).toBe('file: license.txt') + expect(license.text.contentType).toBe('text/plain') + expect(license.text.encoding).toBe('base64') + expect(license.text.content).toBe('license file content') + }) + it('fetches nothing from directory', async () => { + const fsMock = jest.spyOn(fs, 'readdirSync') + fsMock.mockReturnValue(['nothing.txt']) + const licenses = bomBuilder.fetchLicenseEvidence('test_module') + const license = licenses.next().value + expect(license).toBeFalsy() + }) + }) +})