fix: forceignore ignores output file correctly - cleanup extra type d… (

#1295)

* fix: forceignore ignores output file correctly - cleanup extra type declarations

* fix: check forcignore for CSB with -m

* style: disable complexity for CSB.build

* test: add stream snapshot test

* test: fix decsribe

src/client/metadataApiRetrieve.ts

@@ -119,7 +119,7 @@ export class MetadataApiRetrieve extends MetadataTransfer<
   MetadataApiRetrieveOptions
 > {
   public static DEFAULT_OPTIONS: Partial<MetadataApiRetrieveOptions> = { merge: false };
-  private options: MetadataApiRetrieveOptions;
+  private readonly options: MetadataApiRetrieveOptions;
   private orgId?: string;
 
   public constructor(options: MetadataApiRetrieveOptions) {

src/collections/componentSetBuilder.ts

@@ -6,7 +6,7 @@
  */
 
 import * as path from 'node:path';
-import { StateAggregator, Logger, SfError, Messages } from '@salesforce/core';
+import { Logger, Messages, SfError, StateAggregator } from '@salesforce/core';
 import fs from 'graceful-fs';
 import minimatch from 'minimatch';
 import { MetadataComponent } from '../resolve/types';
@@ -15,6 +15,7 @@ import { ComponentSet } from '../collections/componentSet';
 import { RegistryAccess } from '../registry/registryAccess';
 import type { FileProperties } from '../client/types';
 import { MetadataType } from '../registry/types';
+import { MetadataResolver } from '../resolve';
 import { DestructiveChangesType, FromConnectionOptions } from './types';
 
 Messages.importMessagesDirectory(__dirname);
@@ -69,6 +70,7 @@ export class ComponentSetBuilder {
    * @param options: options for creating a ComponentSet
    */
 
+  // eslint-disable-next-line complexity
   public static async build(options: ComponentSetOptions): Promise<ComponentSet> {
     const logger = Logger.childFromRoot('componentSetBuilder');
     let componentSet: ComponentSet | undefined;
@@ -148,7 +150,21 @@ export class ComponentSetBuilder {
           include: componentSetFilter,
           registry: registryAccess,
         });
-        componentSet.forceIgnoredPaths = resolvedComponents.forceIgnoredPaths;
+
+        if (resolvedComponents.forceIgnoredPaths) {
+          // if useFsForceIgnore = true, then we won't be able to resolve a forceignored path,
+          // which we need to do to get the ignored source component
+          const resolver = new MetadataResolver(registryAccess, undefined, false);
+
+          for (const ignoredPath of resolvedComponents.forceIgnoredPaths ?? []) {
+            resolver.getComponentsFromPath(ignoredPath).map((ignored) => {
+              componentSet = componentSet?.filter(
+                (resolved) => !(resolved.fullName === ignored.name && resolved.type === ignored.type)
+              );
+            });
+          }
+        }
+
         resolvedComponents.toArray().map(addToComponentSet(componentSet));
       }
 
@@ -162,7 +178,7 @@ export class ComponentSetBuilder {
           }`
         );
 
-        const mdMap: MetadataMap = metadata
+        const mdMap = metadata
           ? buildMapFromComponents(metadata.metadataEntries.map(entryToTypeAndName(registryAccess)))
           : (new Map() as MetadataMap);
 
@@ -317,7 +333,7 @@ const typeAndNameToMetadataComponents =
 
 // TODO: use Map.groupBy when it's available
 const buildMapFromComponents = (components: MetadataTypeAndMetadataName[]): MetadataMap => {
-  const mdMap: MetadataMap = new Map();
+  const mdMap: MetadataMap = new Map<string, string[]>();
   components.map((cmp) => {
     mdMap.set(cmp.type.name, [...(mdMap.get(cmp.type.name) ?? []), cmp.metadataName]);
   });

src/convert/metadataConverter.ts

@@ -183,9 +183,7 @@ function getMergeConfigOutputs(
     mergeSet.add(component.parent ?? component);
   }
   const writer = new StandardWriter(output.defaultDirectory);
-  if (output.forceIgnoredPaths) {
-    writer.forceIgnoredPaths = output.forceIgnoredPaths;
-  }
+
   return {
     writer,
     mergeSet,

src/convert/streams.ts

@@ -4,7 +4,7 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-import { basename, dirname, isAbsolute, join } from 'node:path';
+import { isAbsolute, join } from 'node:path';
 import { pipeline as cbPipeline, Readable, Stream, Transform, Writable } from 'node:stream';
 import { promisify } from 'node:util';
 import { Messages, SfError } from '@salesforce/core';
@@ -20,6 +20,7 @@ import { ComponentSet } from '../collections/componentSet';
 import { RegistryAccess } from '../registry/registryAccess';
 import { ensureFileExists } from '../utils/fileSystemHandler';
 import { ComponentStatus, FileResponseSuccess } from '../client/types';
+import { ForceIgnore } from '../resolve';
 import { MetadataTransformerFactory } from './transformers/metadataTransformerFactory';
 import { ConvertContext } from './convertContext/convertContext';
 import { SfdxFileFormat, WriteInfo, WriterFormat } from './types';
@@ -35,7 +36,6 @@ export const stream2buffer = async (stream: Stream): Promise<Buffer> =>
     const buf = Array<any>();
     stream.on('data', (chunk) => buf.push(chunk));
     stream.on('end', () => resolve(Buffer.concat(buf)));
-    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
     stream.on('error', (err) => reject(`error converting stream - ${err}`));
   });
 
@@ -113,7 +113,6 @@ export class ComponentConverter extends Transform {
 }
 
 export abstract class ComponentWriter extends Writable {
-  public forceIgnoredPaths: Set<string> = new Set<string>();
   protected rootDestination?: SourcePath;
   protected logger: Logger;
 
@@ -128,9 +127,11 @@ export class StandardWriter extends ComponentWriter {
   /** filepaths that converted files were written to */
   public readonly converted: string[] = [];
   public readonly deleted: FileResponseSuccess[] = [];
+  public readonly forceignore: ForceIgnore;
 
   public constructor(rootDestination: SourcePath) {
     super(rootDestination);
+    this.forceignore = ForceIgnore.findAndCreate(rootDestination);
   }
 
   public async _write(chunk: WriterFormat, encoding: string, callback: (err?: Error) => void): Promise<void> {
@@ -144,8 +145,7 @@ export class StandardWriter extends ComponentWriter {
         await Promise.all(
           chunk.writeInfos
             .map(makeWriteInfoAbsolute(this.rootDestination))
-            .filter(existsOrDoesntMatchIgnored(this.forceIgnoredPaths))
-            .filter((info) => !this.forceIgnoredPaths.has(info.output))
+            .filter(existsOrDoesntMatchIgnored(this.forceignore))
             .map((info) => {
               if (info.shouldDelete) {
                 this.deleted.push({
@@ -288,11 +288,6 @@ const makeWriteInfoAbsolute =
   });
 
 const existsOrDoesntMatchIgnored =
-  (ignoredPaths: Set<string>) =>
+  (forceignore: ForceIgnore) =>
   (writeInfo: WriteInfo): boolean =>
-    existsSync(writeInfo.output) ||
-    [...ignoredPaths].every(
-      (ignoredPath) =>
-        !dirname(ignoredPath).includes(dirname(writeInfo.output)) &&
-        !basename(ignoredPath).includes(basename(writeInfo.output))
-    );
+    existsSync(writeInfo.output) || forceignore.accepts(writeInfo.output);

src/resolve/adapters/baseSourceAdapter.ts

@@ -35,8 +35,8 @@ export abstract class BaseSourceAdapter implements SourceAdapter {
   public constructor(
     type: MetadataType,
     registry = new RegistryAccess(),
-    forceIgnore: ForceIgnore = new ForceIgnore(),
-    tree: TreeContainer = new NodeFSTreeContainer()
+    forceIgnore = new ForceIgnore(),
+    tree = new NodeFSTreeContainer()
   ) {
     this.type = type;
     this.registry = registry;

src/resolve/forceIgnore.ts

@@ -17,7 +17,7 @@ export class ForceIgnore {
 
   private readonly parser?: Ignore;
   private readonly forceIgnoreDirectory?: string;
-  private DEFAULT_IGNORE: string[] = ['**/*.dup', '**/.*', '**/package2-descriptor.json', '**/package2-manifest.json'];
+  private DEFAULT_IGNORE = ['**/*.dup', '**/.*', '**/package2-descriptor.json', '**/package2-manifest.json'];
 
   public constructor(forceIgnorePath = '') {
     try {

src/resolve/metadataResolver.ts

@@ -31,6 +31,7 @@ export class MetadataResolver {
   /**
    * @param registry Custom registry data
    * @param tree `TreeContainer` to traverse with
+   * @param useFsForceIgnore false = use default forceignore entries, true = search and use forceignore in project
    */
   public constructor(
     private registry = new RegistryAccess(),
@@ -186,7 +187,7 @@ const isProbablyPackageManifest =
  * If a type can be determined from a directory path, and the end part of the path isn't
  * the directoryName of the type itself, infer the path is part of a mixedContent component
  *
- * @param dirPath Path to a directory
+ * @param registry the registry to resolve a type against
  */
 const resolveDirectoryAsComponent =
   (registry: RegistryAccess) =>
@@ -227,8 +228,8 @@ const isMetadata =
  * Attempt to find similar types for types that could not be inferred
  * To be used after executing the resolveType() method
  *
- * @param fsPath
  * @returns an array of suggestions
+ * @param registry a metdata registry to resolve types against
  */
 const getSuggestionsForUnresolvedTypes =
   (registry: RegistryAccess) =>
@@ -350,7 +351,7 @@ const resolveType =
 /**
  * Any file with a registered suffix is potentially a content metadata file.
  *
- * @param fsPath File path of a potential content metadata file
+ * @param registry a metadata registry to resolve types agsinst
  */
 const parseAsContentMetadataXml =
   (registry: RegistryAccess) =>

src/resolve/sourceComponent.ts

@@ -45,8 +45,8 @@ export class SourceComponent implements MetadataComponent {
   public parentType?: MetadataType;
   public content?: string;
   public replacements?: Record<string, MarkedReplacement[]>;
-  private treeContainer: TreeContainer;
-  private forceIgnore: ForceIgnore;
+  private readonly treeContainer: TreeContainer;
+  private readonly forceIgnore: ForceIgnore;
   private markedForDelete = false;
   private destructiveChangesType?: DestructiveChangesType;
 

test/collections/componentSetBuilder.test.ts

@@ -6,6 +6,7 @@
  */
 
 import * as path from 'node:path';
+import { join } from 'node:path';
 import fs from 'graceful-fs';
 import * as sinon from 'sinon';
 import { assert, expect, config } from 'chai';
@@ -14,6 +15,7 @@ import { RegistryAccess } from '../../src/registry/registryAccess';
 import { ComponentSetBuilder, entryToTypeAndName } from '../../src/collections/componentSetBuilder';
 import { ComponentSet } from '../../src/collections/componentSet';
 import { FromSourceOptions } from '../../src/collections/types';
+import { MetadataResolver, SourceComponent } from '../../src';
 
 config.truncateThreshold = 0;
 
@@ -302,6 +304,50 @@ describe('ComponentSetBuilder', () => {
       expect(compSet.has(customObjectComponent)).to.equal(true);
       expect(compSet.has({ type: 'CustomObject', fullName: '*' })).to.equal(true);
     });
+    it('should create ComponentSet from multiple metadata (ApexClass:MyClass,CustomObject), one of which is forceignored', async () => {
+      const customObjSourceComponent = new SourceComponent({
+        name: 'myCO',
+        content: join('my', 'path', 'to', 'a', 'customobject.xml'),
+        parentType: undefined,
+        type: { id: 'customobject', directoryName: 'objects', name: 'CustomObject' },
+        xml: '',
+      });
+
+      componentSet.add(apexClassComponent);
+      componentSet.add(customObjSourceComponent);
+
+      componentSet.add(apexClassWildcardMatch);
+      componentSet.forceIgnoredPaths = new Set<string>();
+      componentSet.forceIgnoredPaths.add(join('my', 'path', 'to', 'a', 'customobject.xml'));
+      fromSourceStub.returns(componentSet);
+      const packageDir1 = path.resolve('force-app');
+
+      sandbox.stub(MetadataResolver.prototype, 'getComponentsFromPath').returns([customObjSourceComponent]);
+
+      const compSet = await ComponentSetBuilder.build({
+        sourcepath: undefined,
+        manifest: undefined,
+        metadata: {
+          metadataEntries: ['ApexClass:MyClass', 'ApexClass:MyClassIsAwesome', 'CustomObject:myCO'],
+          directoryPaths: [packageDir1],
+        },
+      });
+      expect(fromSourceStub.callCount).to.equal(1);
+      const fromSourceArgs = fromSourceStub.firstCall.args[0] as FromSourceOptions;
+      expect(fromSourceArgs).to.have.deep.property('fsPaths', [packageDir1]);
+      const filter = new ComponentSet();
+      filter.add({ type: 'ApexClass', fullName: 'MyClass' });
+      filter.add({ type: 'ApexClass', fullName: 'MyClassIsAwesome' });
+      filter.add({ type: 'CustomObject', fullName: 'myCO' });
+      assert(fromSourceArgs.include instanceof ComponentSet, 'include should be a ComponentSet');
+      expect(fromSourceArgs.include.getSourceComponents()).to.deep.equal(filter.getSourceComponents());
+      expect(compSet.size).to.equal(3);
+      expect(compSet.has(apexClassComponent)).to.equal(true);
+      expect(compSet.has(customObjectComponent)).to.equal(false);
+      expect(compSet.has({ type: 'CustomObject', fullName: '*' })).to.equal(false);
+      expect(compSet.has({ type: 'ApexClass', fullName: 'MyClass' })).to.equal(true);
+      expect(compSet.has({ type: 'ApexClass', fullName: 'MyClassIsAwesome' })).to.equal(true);
+    });
 
     it('should create ComponentSet from partial-match fullName (ApexClass:Prop*)', async () => {
       componentSet.add(apexClassComponent);

test/mock/type-constants/staticresourceConstant.ts

@@ -22,7 +22,7 @@ export const MIXED_CONTENT_DIRECTORY_SOURCE_PATHS = [
   join(MIXED_CONTENT_DIRECTORY_CONTENT_PATH, 'tests', 'test.js'),
   join(MIXED_CONTENT_DIRECTORY_CONTENT_PATH, 'tests', 'test2.pdf'),
 ];
-export const MIXED_CONTENT_DIRECTORY_COMPONENT: SourceComponent = new SourceComponent({
+export const MIXED_CONTENT_DIRECTORY_COMPONENT = new SourceComponent({
   name: 'aStaticResource',
   type,
   xml: MIXED_CONTENT_DIRECTORY_XML_PATHS[0],

test/snapshot/sampleProjects/forceignore/.forceignore

@@ -0,0 +1 @@
+**/*.profile

test/snapshot/sampleProjects/forceignore/__snapshots__/ignores-source-format-path.expected/testOutput/source-format/main/default/classes/OneClass.cls

@@ -0,0 +1,5 @@
+public with sharing class OneClass {
+    public OneClass() {
+
+    }
+}

test/snapshot/sampleProjects/forceignore/__snapshots__/ignores-source-format-path.expected/testOutput/source-format/main/default/classes/OneClass.cls-meta.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ApexClass xmlns="http://soap.sforce.com/2006/04/metadata">
+    <apiVersion>59.0</apiVersion>
+    <status>Active</status>
+</ApexClass>

test/snapshot/sampleProjects/forceignore/originalMdapi/classes/OneClass.cls

@@ -0,0 +1,5 @@
+public with sharing class OneClass {
+    public OneClass() {
+
+    }
+}

test/snapshot/sampleProjects/forceignore/originalMdapi/classes/OneClass.cls-meta.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ApexClass xmlns="http://soap.sforce.com/2006/04/metadata">
+    <apiVersion>59.0</apiVersion>
+    <status>Active</status>
+</ApexClass>

test/snapshot/sampleProjects/forceignore/originalMdapi/package.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Package xmlns="http://soap.sforce.com/2006/04/metadata">
+    <types>
+        <members>OneClass</members>
+        <name>ApexClass</name>
+    </types>
+    <types>
+        <members>Admin</members>
+        <name>Profile</name>
+    </types>
+    <version>60.0</version>
+</Package>

test/snapshot/sampleProjects/forceignore/originalMdapi/profiles/Admin.profile

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Profile xmlns="http://soap.sforce.com/2006/04/metadata">
+    <custom>false</custom>
+    <userLicense>Salesforce</userLicense>
+    <userPermissions>
+        <enabled>true</enabled>
+        <name>AIViewInsightObjects</name>
+    </userPermissions>
+</Profile>

test/snapshot/sampleProjects/forceignore/sfdx-project.json

@@ -0,0 +1,12 @@
+{
+  "name": "forceignore",
+  "namespace": "",
+  "packageDirectories": [
+    {
+      "default": true,
+      "path": "force-app"
+    }
+  ],
+  "sfdcLoginUrl": "https://login.salesforce.com",
+  "sourceApiVersion": "52.0"
+}