Merge branch 'main' into feat/tools-as-components

.github/workflows/nodejs.yml

@@ -43,7 +43,7 @@ jobs:
       - name: setup subject
         run: npm i --ignore-scripts --loglevel=silly
       - name: build
-        run: npm run build
+        run: npm run build-dev
       - name: artifact build result
         # see https://github.com/actions/upload-artifact
         uses: actions/upload-artifact@v4

CONTRIBUTING.md

@@ -16,7 +16,7 @@ npm ci
 ## Build from source
 
 ```shell
-npm run build
+npm run build-dev
 ```
 
 ## Testing

HISTORY.md

@@ -6,14 +6,20 @@ All notable changes to this project will be documented in this file.
 
 <!-- unreleased changes go here -->
 
+* Added
+  * Capability to gather license text evidences (#256 via #1243)  
+    This feature can be controlled via CLI switch `--gather-license-texts`.  
+    This feature is experimental. This feature is disabled per default.
 * Dependencies
-  * No longer depend on `packageurlk-js` (via [#1237])
+  * No longer depend on `packageurl-js` (via [#1237])
 * Build
   * Use _TypeScript_ `v5.6.2` now, was `v5.5.3` (via [#1209], [#1218])
 
+[#256]: https://github.com/CycloneDX/cyclonedx-node-npm/issues/256
 [#1209]: https://github.com/CycloneDX/cyclonedx-node-npm/pull/1209
 [#1218]: https://github.com/CycloneDX/cyclonedx-node-npm/pull/1218
 [#1237]: https://github.com/CycloneDX/cyclonedx-node-npm/pull/1237
+[#1243]: https://github.com/CycloneDX/cyclonedx-node-npm/pull/1243
 
 ## 1.19.3 -- 2024-07-15
 

README.md

@@ -87,6 +87,8 @@ Options:
   --omit <type...>          Dependency types to omit from the installation tree.
                             (can be set multiple times)
                             (choices: "dev", "optional", "peer", default: "dev" if the NODE_ENV environment variable is set to "production", otherwise empty)
+  --gather-license-texts    Search for license files in components and include them as license evidence.
+                            This feature is experimental. (default: false)
   --flatten-components      Whether to flatten the components.
                             This means the actual nesting of node packages is not represented in the SBOM result.
                             (default: false)

package.json

@@ -91,6 +91,7 @@
     "lint": "tsc --noEmit",
     "prebuild": "node -r fs -e 'fs.rmSync(`dist`,{recursive:true,force:true})'",
     "build": "tsc -b ./tsconfig.json",
+    "build-dev": "npm run -- build --sourceMap",
     "cs-fix": "eslint --fix .",
     "setup-tests": "node tests/integration/setup.js",
     "test": "run-p --aggregate-output -lc test:*",

src/_helpers.ts

@@ -18,6 +18,7 @@ Copyright (c) OWASP Foundation. All Rights Reserved.
 */
 
 import { readFileSync, writeSync } from 'fs'
+import { extname, parse } from 'path'
 
 export function loadJsonFile (path: string): any {
   return JSON.parse(readFileSync(path, 'utf8'))
@@ -56,3 +57,46 @@ export function tryRemoveSecretsFromUrl (url: string): string {
     return url
   }
 }
+
+// region MIME
+
+export type MimeType = string
+
+const MIME_TEXT_PLAIN: MimeType = 'text/plain'
+
+const MAP_TEXT_EXTENSION_MIME: Readonly<Record<string, MimeType>> = {
+  '': MIME_TEXT_PLAIN,
+  // https://www.iana.org/assignments/media-types/media-types.xhtml
+  '.csv': 'text/csv',
+  '.htm': 'text/html',
+  '.html': 'text/html',
+  '.md': 'text/markdown',
+  '.txt': MIME_TEXT_PLAIN,
+  '.rst': 'text/prs.fallenstein.rst',
+  '.xml': 'text/xml', // not `application/xml` -- our scope is text!
+  // add more mime types above this line. pull-requests welcome!
+  // license-specific files
+  '.license': MIME_TEXT_PLAIN,
+  '.licence': MIME_TEXT_PLAIN
+} as const
+
+export function getMimeForTextFile (filename: string): MimeType | undefined {
+  return MAP_TEXT_EXTENSION_MIME[extname(filename).toLowerCase()]
+}
+
+const LICENSE_FILENAME_BASE = new Set(['licence', 'license'])
+const LICENSE_FILENAME_EXT = new Set([
+  '.apache',
+  '.bsd',
+  '.gpl',
+  '.mit'
+])
+
+export function getMimeForLicenseFile (filename: string): MimeType | undefined {
+  const { name, ext } = parse(filename.toLowerCase())
+  return LICENSE_FILENAME_BASE.has(name) && LICENSE_FILENAME_EXT.has(ext)
+    ? MIME_TEXT_PLAIN
+    : MAP_TEXT_EXTENSION_MIME[ext]
+}
+
+// endregion MIME

src/builders.ts

@@ -18,11 +18,12 @@ Copyright (c) OWASP Foundation. All Rights Reserved.
 */
 
 import { type Builders, Enums, type Factories, Models, Utils } from '@cyclonedx/cyclonedx-library'
-import { existsSync } from 'fs'
+import { existsSync, readdirSync, readFileSync } from 'fs'
 import * as normalizePackageData from 'normalize-package-data'
 import * as path from 'path'
+import { join } from 'path'
 
-import { isString, loadJsonFile, tryRemoveSecretsFromUrl } from './_helpers'
+import { getMimeForLicenseFile, isString, loadJsonFile, tryRemoveSecretsFromUrl } from './_helpers'
 import { makeNpmRunner, type runFunc } from './npmRunner'
 import { PropertyNames, PropertyValueBool } from './properties'
 import { versionCompare } from './versionCompare'
@@ -37,6 +38,7 @@ interface BomBuilderOptions {
   reproducible?: BomBuilder['reproducible']
   flattenComponents?: BomBuilder['flattenComponents']
   shortPURLs?: BomBuilder['shortPURLs']
+  gatherLicenseTexts?: BomBuilder['gatherLicenseTexts']
 }
 
 type cPath = string
@@ -55,6 +57,7 @@ export class BomBuilder {
   reproducible: boolean
   flattenComponents: boolean
   shortPURLs: boolean
+  gatherLicenseTexts: boolean
 
   console: Console
 
@@ -76,6 +79,7 @@ export class BomBuilder {
     this.reproducible = options.reproducible ?? false
     this.flattenComponents = options.flattenComponents ?? false
     this.shortPURLs = options.shortPURLs ?? false
+    this.gatherLicenseTexts = options.gatherLicenseTexts ?? false
 
     this.console = console_
   }
@@ -462,6 +466,23 @@ export class BomBuilder {
       l.acknowledgement = Enums.LicenseAcknowledgement.Declared
     })
 
+    if (this.gatherLicenseTexts) {
+      if (this.packageLockOnly) {
+        this.console.warn('WARN  | Adding license text is ignored (package-lock-only is configured!) for %j', data.name)
+      } else {
+        component.evidence = new Models.ComponentEvidence()
+        for (const license of this.fetchLicenseEvidence(data?.path as string)) {
+          if (license != null) {
+            // only create a evidence if a license attachment is found
+            if (component.evidence == null) {
+              component.evidence = new Models.ComponentEvidence()
+            }
+            component.evidence.licenses.add(license)
+          }
+        }
+      }
+    }
+
     if (isOptional || isDevOptional) {
       component.scope = Enums.ComponentScope.Optional
     }
@@ -609,6 +630,35 @@ export class BomBuilder {
       }
     }
   }
+
+  readonly #LICENSE_FILENAME_PATTERN = /^(?:UN)?LICEN[CS]E|.\.LICEN[CS]E$|^NOTICE$/i
+
+  private * fetchLicenseEvidence (path: string): Generator<Models.License | null, void, void> {
+    const files = readdirSync(path)
+    for (const file of files) {
+      if (!this.#LICENSE_FILENAME_PATTERN.test(file)) {
+        continue
+      }
+
+      const contentType = getMimeForLicenseFile(file)
+      if (contentType === undefined) {
+        continue
+      }
+
+      const fp = join(path, file)
+      yield new Models.NamedLicense(
+        `file: ${file}`,
+        {
+          text: new Models.Attachment(
+            readFileSync(fp).toString('base64'),
+            {
+              contentType,
+              encoding: Enums.AttachmentEncoding.Base64
+            }
+          )
+        })
+    }
+  }
 }
 
 class DummyComponent extends Models.Component {

src/cli.ts

@@ -43,10 +43,11 @@ interface CommandOptions {
   ignoreNpmErrors: boolean
   packageLockOnly: boolean
   omit: Omittable[]
-  specVersion: Spec.Version
+  gatherLicenseTexts: boolean
   flattenComponents: boolean
   shortPURLs: boolean
   outputReproducible: boolean
+  specVersion: Spec.Version
   outputFormat: OutputFormat
   outputFile: string
   validate: boolean | undefined
@@ -86,6 +87,12 @@ function makeCommand (process: NodeJS.Process): Command {
         : [],
       `"${Omittable.Dev}" if the NODE_ENV environment variable is set to "production", otherwise empty`
     )
+  ).addOption(
+    new Option(
+      '--gather-license-texts',
+      'Search for license files in components and include them as license evidence.\n' +
+      'This feature is experimental.'
+    ).default(false)
   ).addOption(
     new Option(
       '--flatten-components',
@@ -244,6 +251,7 @@ export async function run (process: NodeJS.Process): Promise<number> {
       metaComponentType: options.mcType,
       packageLockOnly: options.packageLockOnly,
       omitDependencyTypes: options.omit,
+      gatherLicenseTexts: options.gatherLicenseTexts,
       reproducible: options.outputReproducible,
       flattenComponents: options.flattenComponents,
       shortPURLs: options.shortPURLs

tests/_data/dummy_projects/with-prepared/.gitattributes

@@ -0,0 +1,3 @@
+
+*     linguist-vendored -text
+**/*  linguist-vendored -text

tests/_data/dummy_projects/with-prepared/.gitignore

@@ -0,0 +1,13 @@
+*
+!/.gitignore
+!/.gitattributes
+
+!/node_modules
+!/node_modules/*
+!/node_modules/**/*
+!/package.json
+!/package-lock.json
+
+!/setup.sh
+!/README.md
+!/LICENCE.mit

tests/_data/dummy_projects/with-prepared/LICENCE.mit

@@ -0,0 +1 @@
+this is a dummy license file.

tests/_data/dummy_projects/with-prepared/README.md

@@ -0,0 +1,10 @@
+# setup with prepared data and structure
+
+this setup actually vendors some 3rd-party data in the `node_modules` dir:
+- package manifests
+- license files
+
+this dir is already set up as expected.
+no need to run `npm install`.
+
+if this setup is to be changed or recreated, run `./setup.sh`.