Fix container scanner paths when a different mountpoint is used

.gitignore

@@ -16,4 +16,4 @@ coverage/
 
 resources/
 client/resources/
-integration-tests/project-folder/build
+integration-tests/project-folder/build*

client/README.md

@@ -26,8 +26,9 @@ Or, if `pathToEnvScript` and `pathToBuildFolder` are defined:
 ```shell
  $ $commandWrapper ". $pathToEnvScript $pathToBuildFolder && bitbake core-image-minimal"
 ```
+You can also control the directory from which they are started by using the `bitbake.workingDirectory` option as well as the shell environment variables with `bitbake.shellEnv`.
 
-Here are some examples using the most popular bitbake wrappers. You can also control the directory from which they are started by using the `bitbake.workingDirectory` option:
+Here are some examples using the most popular bitbake wrappers:
 ```json
 {
     "bitbake.commandWrapper": "docker run --rm -v ${workspaceFolder}:${workspaceFolder} crops/poky --workdir=${workspaceFolder} /bin/bash -c",
@@ -38,10 +39,40 @@ Here are some examples using the most popular bitbake wrappers. You can also con
     "bitbake.commandWrapper": "kas shell -c",
     "bitbake.workingDirectory": "${workspaceFolder}/yocto"
 }
-{ "bitbake.commandWrapper": "cqfd run" }
+{
+    "bitbake.commandWrapper": "cqfd run",
+    "bitbake.shellEnv": {
+        "CQFD_EXTRA_RUN_ARGS": "-e DISPLAY=:0 -v /tmp/.X11-unix:/tmp/.X11-unix"
+    }
+}
 { "bitbake.commandWrapper": "${workspaceFolder}/build.sh --" }
 ```
 
+### Additional settings recommendations
+
+If your workspace contains a Yocto build directory, some other extensions may
+be hogging lots of resources to parse it's contents and stall your machine. Here are
+some example settings to improve your experience (assuming your build directory
+is called `build`). You can add them to your `settings.json` file, or your global
+user settings.
+
+```json
+{
+    "files.watcherExclude": {
+        "**/build/**": true
+    },
+    "search.exclude": {
+        "**/build/**": true
+    },
+    "C_Cpp.files.exclude": {
+        "**/build": true
+    },
+    "python.analysis.exclude": [
+        "**/build/**"
+    ]
+}
+```
+
 ## Features
 
 ### Syntax highlighting

client/package.json

@@ -124,6 +124,11 @@
           "type": "string",
           "default": "${workspaceFolder}",
           "description": "Set the working directory for running BitBake command."
+        },
+        "bitbake.shellEnv": {
+          "type": "object",
+          "default": {},
+          "description": "Environment variables to set before running the BitBake command."
         }
       }
     },

client/src/driver/BitBakeProjectScanner.ts

@@ -20,6 +20,7 @@ import type {
 
 import { type BitbakeDriver } from './BitbakeDriver'
 import { type LanguageClient } from 'vscode-languageclient/node'
+import fs from 'fs'
 
 interface ScannStatus {
   scanIsRunning: boolean
@@ -40,6 +41,10 @@ export class BitBakeProjectScanner {
   private _bitbakeDriver: BitbakeDriver | undefined
   private _languageClient: LanguageClient | undefined
 
+  /// These attributes map bind mounts of the workDir to the host system if a docker container commandWrapper is used (-v).
+  private containerMountPoint: string | undefined
+  private hostMountPoint: string | undefined
+
   setDriver (bitbakeDriver: BitbakeDriver): void {
     this._bitbakeDriver = bitbakeDriver
   }
@@ -109,6 +114,47 @@ export class BitBakeProjectScanner {
     }
   }
 
+  private async getContainerParentInodes (filepath: string): Promise<number[]> {
+    const commandResult = await this.executeBitBakeCommand(`f=${filepath}; while [[ $f != / ]]; do stat -c %i $f; f=$(dirname "$f"); done;`)
+    const stdout = commandResult.stdout.toString().trim()
+    const regex = /^\d+$/gm
+    const matches = stdout.match(regex)
+    return (matches != null) ? matches.map((match) => parseInt(match)) : [NaN]
+  }
+
+  /// Find corresponding mount point inode in layerPath/hostWorkdir and all parents
+  private async scanContainerMountPoint (layerPath: string, hostWorkdir: string): Promise<void> {
+    this.containerMountPoint = undefined
+    this.hostMountPoint = undefined
+
+    if (fs.existsSync(layerPath)) {
+      // We're not inside a container, or the container is not using a different workdir
+      return
+    }
+
+    let containerDir = layerPath
+    const containerDirInodes = await this.getContainerParentInodes(containerDir)
+    let hostDir = hostWorkdir
+
+    while (hostDir !== '/') {
+      const hostDirInode = fs.statSync(hostDir).ino
+
+      let containerIdx = 0
+      while (containerDir !== '/') {
+        const containerDirInode = containerDirInodes[containerIdx]
+        logger.debug('Comparing container inodes: ' + containerDir + ':' + containerDirInode + ' ' + hostDir + ':' + hostDirInode)
+        if (containerDirInode === hostDirInode) {
+          this.containerMountPoint = containerDir
+          this.hostMountPoint = hostDir
+          return
+        }
+        containerDir = path.dirname(containerDir)
+        containerIdx++
+      }
+      hostDir = path.dirname(hostDir)
+    }
+  }
+
   private printScanStatistic (): void {
     logger.info('Scan results:')
     logger.info('******************************************************************')
@@ -129,6 +175,7 @@ export class BitBakeProjectScanner {
 
   private async scanAvailableLayers (): Promise<void> {
     this._bitbakeScanResult._layers = new Array < LayerInfo >()
+    this.containerMountPoint = undefined
 
     const commandResult = await this.executeBitBakeCommand('bitbake-layers show-layers')
 
@@ -145,15 +192,15 @@ export class BitBakeProjectScanner {
       }
 
       for (const element of outputLines.slice(layersFirstLine + 2)) {
-        const tempElement: string[] = element.split(/\s+/)
+        const tempElement = element.split(/\s+/)
         const layerElement = {
           name: tempElement[0],
-          path: tempElement[1],
+          path: await this.resolveContainerPath(tempElement[1]),
           priority: parseInt(tempElement[2])
         }
 
-        if ((layerElement.name !== undefined) && (layerElement.path !== undefined) && layerElement.priority !== undefined) {
-          this._bitbakeScanResult._layers.push(layerElement)
+        if ((layerElement.name !== undefined) && (layerElement.path !== undefined) && (layerElement.priority !== undefined)) {
+          this._bitbakeScanResult._layers.push(layerElement as LayerInfo)
         }
       }
     } else {
@@ -162,6 +209,26 @@ export class BitBakeProjectScanner {
     }
   }
 
+  /// If a docker container is used, the workdir may be different from the host system.
+  /// This function resolves the path to the host system.
+  private async resolveContainerPath (layerPath: string | undefined): Promise<string | undefined> {
+    if (layerPath === undefined) {
+      return undefined
+    }
+    const hostWorkdir = this.bitbakeDriver?.bitbakeSettings.workingDirectory
+    if (hostWorkdir === undefined) {
+      throw new Error('hostWorkdir is undefined')
+    }
+    if (this.containerMountPoint === undefined) {
+      await this.scanContainerMountPoint(layerPath, hostWorkdir)
+    }
+    if (this.containerMountPoint === undefined || this.hostMountPoint === undefined) {
+      return layerPath
+    }
+    const relativePath = path.relative(this.containerMountPoint, layerPath)
+    return path.resolve(this.hostMountPoint, relativePath)
+  }
+
   private searchFiles (pattern: string): ElementInfo[] {
     const elements: ElementInfo[] = new Array < ElementInfo >()
 

client/src/driver/BitbakeDriver.ts

@@ -39,7 +39,8 @@ export class BitbakeDriver {
     this.bitbakeActive = true
     const child = childProcess.spawn(script, {
       shell,
-      cwd: this.bitbakeSettings.workingDirectory
+      cwd: this.bitbakeSettings.workingDirectory,
+      env: { ...process.env, ...this.bitbakeSettings.shellEnv }
     })
     child.on('close', () => {
       this.bitbakeActive = false
@@ -55,7 +56,8 @@ export class BitbakeDriver {
     this.bitbakeActive = true
     const ret = childProcess.spawnSync(script, {
       shell,
-      cwd: this.bitbakeSettings.workingDirectory
+      cwd: this.bitbakeSettings.workingDirectory,
+      env: { ...process.env, ...this.bitbakeSettings.shellEnv }
     })
     this.bitbakeActive = false
     return ret
@@ -109,7 +111,7 @@ export class BitbakeDriver {
     }
 
     const pathToEnvScript = this.bitbakeSettings.pathToEnvScript
-    if (pathToEnvScript !== undefined && !fs.existsSync(pathToEnvScript)) {
+    if (this.bitbakeSettings.commandWrapper === undefined && pathToEnvScript !== undefined && !fs.existsSync(pathToEnvScript)) {
       clientNotificationManager.showBitbakeError("Bitbake environment script doesn't exist: " + pathToEnvScript)
       return false
     }

client/src/lib/src/BitbakeSettings.ts

@@ -12,6 +12,7 @@ export interface BitbakeSettings {
   pathToEnvScript?: string
   commandWrapper?: string
   workingDirectory: string
+  shellEnv?: NodeJS.Dict<string>
 }
 
 export function loadBitbakeSettings (settings: any, workspaceFolder: string): BitbakeSettings {
@@ -29,7 +30,8 @@ export function loadBitbakeSettings (settings: any, workspaceFolder: string): Bi
     pathToBuildFolder: settings.pathToBuildFolder !== '' ? resolveSettingsPath(settings.pathToBuildFolder, workspaceFolder) : undefined,
     pathToEnvScript: settings.pathToEnvScript !== '' ? resolveSettingsPath(settings.pathToEnvScript, workspaceFolder) : undefined,
     commandWrapper: settings.commandWrapper !== '' ? expandWorkspaceFolder(settings.commandWrapper, workspaceFolder) : undefined,
-    workingDirectory: settings.workingDirectory !== '' ? resolveSettingsPath(settings.workingDirectory, workspaceFolder) : workspaceFolder
+    workingDirectory: settings.workingDirectory !== '' ? resolveSettingsPath(settings.workingDirectory, workspaceFolder) : workspaceFolder,
+    shellEnv: toStringDict(settings.shellEnv)
   }
 }
 
@@ -48,3 +50,7 @@ function sanitizeForShell (command: string | undefined): string | undefined {
   }
   return command.replace(/[;`&|<>\\$(){}!#*?"']/g, '')
 }
+
+function toStringDict (dict: object | undefined): NodeJS.Dict<string> | undefined {
+  return dict as NodeJS.Dict<string> | undefined
+}

integration-tests/project-folder/sources/meta-fixtures/definition.bb

@@ -0,0 +1 @@
+inherit image

integration-tests/src/tests/command-wrapper.test.ts

@@ -0,0 +1,74 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) 2023 Savoir-faire Linux. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import * as assert from 'assert'
+import * as vscode from 'vscode'
+
+import { delay } from '../utils/async'
+import path from 'path'
+
+suite('Bitbake Command Wrapper', () => {
+  let workspaceURI: vscode.Uri
+  let buildFolder: vscode.Uri
+  let savedSettings: {
+    pathToBuildFolder: string | undefined
+    pathToEnvScript: string | undefined
+  }
+
+  suiteSetup(async function (this: Mocha.Context) {
+    /* eslint-disable no-template-curly-in-string */
+    this.timeout(100000)
+    while (vscode.workspace.workspaceFolders === undefined || vscode.workspace.workspaceFolders?.length === 0) {
+      await delay(100)
+    }
+    workspaceURI = vscode.workspace.workspaceFolders[0].uri
+    buildFolder = vscode.Uri.joinPath(workspaceURI, 'build-crops')
+
+    const vscodeBitbake = vscode.extensions.getExtension('yocto-project.yocto-bitbake')
+    if (vscodeBitbake === undefined) {
+      assert.fail('Bitbake extension is not available')
+    }
+    await vscodeBitbake.activate()
+
+    const bitbakeConfiguration = vscode.workspace.getConfiguration('bitbake')
+    savedSettings = {
+      pathToBuildFolder: bitbakeConfiguration.get('pathToBuildFolder'),
+      pathToEnvScript: bitbakeConfiguration.get('pathToEnvScript')
+    }
+
+    // We use purposely complex mount points to test the scanner path resolution logic
+    await bitbakeConfiguration.update('commandWrapper', 'docker run --rm -v ${workspaceFolder}/../..:/workdir/ crops/poky --workdir=/workdir /bin/bash -c')
+    await bitbakeConfiguration.update('pathToBuildFolder', '/workdir/integration-tests/project-folder/build-crops')
+    await bitbakeConfiguration.update('pathToEnvScript', '/workdir/integration-tests/project-folder/sources/poky/oe-init-build-env')
+  })
+
+  suiteTeardown(async function (this: Mocha.Context) {
+    await vscode.workspace.fs.delete(buildFolder, { recursive: true })
+
+    const bitbakeConfiguration = vscode.workspace.getConfiguration('bitbake')
+    await bitbakeConfiguration.update('commandWrapper', undefined)
+    await bitbakeConfiguration.update('pathToBuildFolder', savedSettings.pathToBuildFolder)
+    await bitbakeConfiguration.update('pathToEnvScript', savedSettings.pathToEnvScript)
+  })
+
+  test('Bitbake can properly scan includes inside a crops container', async () => {
+    const filePath = path.resolve(__dirname, '../../project-folder/sources/meta-fixtures/definition.bb')
+    const docUri = vscode.Uri.parse(`file://${filePath}`)
+    let definitions: vscode.Location[] = []
+
+    await vscode.workspace.openTextDocument(docUri)
+
+    while (definitions.length !== 1) {
+      definitions = await vscode.commands.executeCommand(
+        'vscode.executeDefinitionProvider',
+        docUri,
+        new vscode.Position(0, 10)
+      )
+
+      // We cannot get an event when the BitBake scan is complete, so we have to wait for it to finish and hope for the best
+      await delay(100)
+    }
+  }).timeout(300000)
+})

package.json

@@ -35,7 +35,7 @@
 		"clean:server": "rm -fr ./server/node_modules ./client/server server/tsconfig.tsbuildinfo server/.vscode-test",
 		"clean:client": "rm -fr ./client/node_modules ./client/out client/tsconfig.tsbuildinfo client/.vscode-test client/*.vsix",
 		"clean:lib": "rm -fr ./lib/node_modules",
-		"clean": "npm run clean:lib && npm run clean:server && npm run clean:client && rm -fr node_modules integration-tests/out .vscode-test .eslintcache resources/poky* resources/docs coverage ./out",
+		"clean": "npm run clean:lib && npm run clean:server && npm run clean:client && rm -fr node_modules integration-tests/out integration-tests/project-folder/build* .vscode-test .eslintcache resources/poky* resources/docs coverage ./out",
 		"lint": "eslint . --ext js,ts --cache",
 		"jest": "jest",
 		"test": "npm run jest && npm run test:integration && cd client && npm run test:grammar",