V0.9.12 (#104)

- Allow failure tolerance to be set to 0 on validate-tasks command (allows CI/CD processes to fail on validation)
- Added support for `Mappings` section / `!FindInMap` / `!Select` for task files.
- Added functions `!MD5` / `!ReadFile` that can be used in task files.
- Added function `!JsonString` that can be used in task files.
- Added support for `!Ref OrganizationRoot` (and other types) in task files.
- Fixed bug on `org-formation init` where tags on the MasterAccount where not added to generated template.
- Updating stacks that have state `ROLLBACK_FAILED` will be retried.
- Support for large (> 512000 byte) templates

CHANGELOG.md

@@ -1,13 +1,22 @@
 # Changelog
 All notable changes to aws organization formation will be documented in this file.
 
+**version 0.9.12**
+- Allow failure tolerance to be set to 0 on validate-tasks command (allows CI/CD processes to fail on validation)
+- Added support for `Mappings` section / `!FindInMap` / `!Select` for task files.
+- Added functions `!MD5` / `!ReadFile` that can be used in task files.
+- Added function `!JsonString` that can be used in task files.
+- Added support for `!Ref OrganizationRoot` (and other types) in task files.
+- Fixed bug on `org-formation init` where tags on the MasterAccount where not added to generated template.
+- Updating stacks that have state `ROLLBACK_FAILED` will be retried.
+- Support for large (> 512000 byte) templates
+
 **version 0.9.11**
-- Added pseudo parameter `ORG::PrincipalOrgID`.
+- Added pseudo parameter `ORG::PrincipalOrgID` (in tasks file).
 - Improved parsing of attributes in task files.
 - AWSAccount can be used as alias for `CurrentAccount` in task file expressions.
 - Added support for cross account references on `VPCEndpoint.DnsEntries`.
 
-
 **version 0.9.10**
 - Fixed bug where `register-type` tasks did not properly register execution role.
 

docs/task-files.md

@@ -119,6 +119,37 @@ PolicyTemplate:
     bucketArn3: !CopyValue [BucketArn, 123123123123, 'eu-west-1']
 ```
 
+
+### !ReadFile
+
+The `!ReadFile` function will take 1 string argument, a file path, and return the contents of the file as a string.
+
+
+### !MD5
+
+The `!MD5` function will take 1 argument and return a message digest over its value. If the argument is a string, the function will calculate a message digest over the string. If the value is an object the `!MD5` function will create a message digest over the JSON string representation of the contents.
+
+See the following examples:
+
+``` yaml
+
+CopyFileWithHashInKey:
+  Type: copy-to-s3
+  LocalPath: ./source-file.yml
+  RemotePath: !Sub
+  - 's3://organization-formation-${AWS::AccountId}/remote-path-${hashOfFile}.yml'
+  - { hashOfFile: !MD5 { file: !ReadFile './source-file.yml'}}
+  OrganizationBinding:
+    IncludeMasterAccount: true
+    Region: us-east-1
+
+```
+
+### !JsonString
+
+The `!JsonString` function will take 1 or 2 arguments. The first argument will be converted to a JSON string representation. If the second argument is the literal 'pretty-print', the result will contain whitespace, otherwise the result will not contain whitespace. If the first argument is a string, the string will be first converted to an object (assuming the string as json) prior to returning the string representation (therefore minifying the input string).
+
+
 ## Task types
 
 ### update-organization

package.json

@@ -1,6 +1,6 @@
 {
   "name": "aws-organization-formation",
-  "version": "0.9.11",
+  "version": "0.9.12",
   "description": "Infrastructure as code solution for AWS Organizations",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

src/build-tasks/build-configuration.ts

@@ -8,10 +8,12 @@ import { IUpdateStacksBuildTask } from './tasks/update-stacks-task';
 import { IPerformTasksCommandArgs } from '~commands/index';
 import { yamlParse } from '~yaml-cfn/index';
 import { CfnExpressionResolver } from '~core/cfn-expression-resolver';
+import { CfnMappingsSection } from '~core/cfn-functions/cfn-find-in-map';
 
 export class BuildConfiguration {
     public tasks: IBuildTaskConfiguration[];
     public parameters: Record<string, IBuildFileParameter>;
+    public mappings: CfnMappingsSection;
     private file: string;
 
     constructor(input: string, private readonly parameterValues: Record<string, string> = {}) {
@@ -103,7 +105,9 @@ export class BuildConfiguration {
     }
     public enumBuildConfigurationFromBuildFile(filePath: string, buildFile: IBuildFile): IBuildTaskConfiguration[] {
         this.parameters = buildFile.Parameters;
+        this.mappings = buildFile.Mappings;
         delete buildFile.Parameters;
+        delete buildFile.Mappings;
 
         const expressionResolver = new CfnExpressionResolver();
         for(const paramName in this.parameters) {
@@ -144,7 +148,9 @@ export class BuildConfiguration {
 
             expressionResolver.addParameter(paramName, value);
         }
-        const resolvedContents = expressionResolver.resolveParameters(buildFile);
+        expressionResolver.addMappings(this.mappings);
+        expressionResolver.setFilePath(filePath);
+        const resolvedContents = expressionResolver.resolveFirstPass(buildFile);
 
         const result: IBuildTaskConfiguration[] = [];
         for (const name in resolvedContents) {
@@ -191,6 +197,7 @@ export interface IBuildTask {
 
 export interface IBuildFile extends Record<string, IBuildTaskConfiguration | {}>{
     Parameters?: Record<string, IBuildFileParameter>;
+    Mappings?: CfnMappingsSection;
 }
 
 export interface IBuildFileParameter {

src/build-tasks/tasks/include-task.ts

@@ -68,7 +68,7 @@ export class IncludeTaskProvider implements IBuildTaskProvider<IIncludeTaskConfi
             skip: typeof config.Skip === 'boolean' ? config.Skip : undefined,
             childTasks,
             isDependency: (): boolean => false,
-            perform: async (): Promise<void> => await BuildRunner.RunValidationTasks(childTasks, commandForInclude.verbose === true, 1, 999),
+            perform: async (): Promise<void> => await BuildRunner.RunValidationTasks(childTasks, commandForInclude.verbose === true, config.MaxConcurrentTasks, config.FailedTaskTolerance),
         };
     }
 

src/build-tasks/tasks/organization-task.ts

@@ -3,6 +3,7 @@ import { ConsoleUtil } from '../../util/console-util';
 import { ICommandArgs, IUpdateOrganizationCommandArgs, UpdateOrganizationCommand } from '../../commands/index';
 import { IBuildTask, IBuildTaskConfiguration } from '~build-tasks/build-configuration';
 import { IBuildTaskProvider } from '~build-tasks/build-task-provider';
+import { ValidateOrganizationCommand } from '~commands/validate-organization';
 
 export abstract class BaseOrganizationTask implements IBuildTask {
     public name: string;
@@ -50,8 +51,9 @@ export class UpdateOrganizationTask extends BaseOrganizationTask {
 }
 
 export class ValidateOrganizationTask extends BaseOrganizationTask {
-    protected async innerPerform(/* commandArgs: IUpdateOrganizationCommandArgs */): Promise<void> {
-        // no op.
+    protected async innerPerform(commandArgs: IUpdateOrganizationCommandArgs): Promise<void> {
+        ConsoleUtil.LogInfo(`Executing: ${this.config.Type} ${this.templatePath}.`);
+        await ValidateOrganizationCommand.Perform(commandArgs);
     }
 
 }

src/build-tasks/tasks/update-stacks-task.ts

@@ -39,7 +39,7 @@ export class UpdateStacksBuildTaskProvider implements IBuildTaskProvider<IUpdate
             childTasks: [],
             skip: typeof config.Skip === 'boolean' ? config.Skip : undefined,
             StackName: config.StackName,
-            isDependency: (): boolean => false,
+            isDependency: BuildTaskProvider.createIsDependency(config),
             perform: async (): Promise<void> => {
                 const updateStacksCommand = UpdateStacksBuildTaskProvider.createUpdateStacksCommandArgs(config, command);
                 await ValidateStacksCommand.Perform(updateStacksCommand);

src/cfn-binder/cfn-task-provider.ts

@@ -1,6 +1,6 @@
 import { CreateStackInput, DeleteStackInput, UpdateStackInput } from 'aws-sdk/clients/cloudformation';
 import uuid = require('uuid');
-import { AwsUtil } from '../util/aws-util';
+import { AwsUtil, CfnUtil } from '../util/aws-util';
 import { ConsoleUtil } from '../util/console-util';
 import { OrgFormationError } from '../../src/org-formation-error';
 import { ICfnBinding } from './cfn-binder';
@@ -72,6 +72,8 @@ export class CfnTaskProvider {
 
                 };
 
+                await CfnUtil.UploadTemplateToS3IfTooLarge(stackInput, binding, stackName, this.template.hash);
+
                 if (binding.stackPolicy !== undefined) {
                     stackInput.StackPolicyBody = JSON.stringify(binding.stackPolicy);
                 }
@@ -125,31 +127,7 @@ export class CfnTaskProvider {
                     }
                 }
                 try {
-                    try {
-                        await cfn.updateStack(stackInput).promise();
-                        await cfn.waitFor('stackUpdateComplete', { StackName: stackName, $waiter: { delay: 1, maxAttempts: 60 * 30 } }).promise();
-                    } catch (err) {
-                        if (err && err.code === 'ValidationError' && err.message) {
-                            const message = err.message as string;
-                            if (-1 !== message.indexOf('ROLLBACK_COMPLETE')) {
-                                await cfn.deleteStack({ StackName: stackName, RoleARN: roleArn }).promise();
-                                await cfn.waitFor('stackDeleteComplete', { StackName: stackName, $waiter: { delay: 1 } }).promise();
-                                await cfn.createStack(stackInput).promise();
-                                await cfn.waitFor('stackCreateComplete', { StackName: stackName, $waiter: { delay: 1, maxAttempts: 60 * 30 } }).promise();
-                            } else if (-1 !== message.indexOf('does not exist')) {
-                                await cfn.createStack(stackInput).promise();
-                                await cfn.waitFor('stackCreateComplete', { StackName: stackName, $waiter: { delay: 1, maxAttempts: 60 * 30 } }).promise();
-                            } else if (-1 !== message.indexOf('No updates are to be performed.')) {
-                                // ignore;
-                            } else if (err.code === 'ResourceNotReady') {
-                                ConsoleUtil.LogError('error when executing CloudFormation');
-                            } else {
-                                throw err;
-                            }
-                        } else {
-                            throw err;
-                        }
-                    }
+                    await CfnUtil.UpdateOrCreateStack(cfn, stackInput);
 
                     if (binding.state === undefined && binding.terminationProtection === true) {
                         ConsoleUtil.LogDebug(`Enabling termination protection for stack ${stackName}`, this.logVerbose);

src/cfn-binder/cfn-task-runner.ts

@@ -22,13 +22,13 @@ export class CfnTaskRunner {
         await GenericTaskRunner.RunTasks<ICfnTask>(tasks, delegate);
     }
 
-    public static async ValidateTemplates(tasks: ICfnTask[], logVerbose: boolean): Promise<void> {
+    public static async ValidateTemplates(tasks: ICfnTask[], logVerbose: boolean, maxConcurrentTasks: number, failedTasksTolerance: number): Promise<void> {
 
         const delegate: ITaskRunnerDelegates<ICfnTask> = {
             getName: task => `Stack ${task.stackName} in account ${task.accountId} (${task.region})`,
             getVerb: () => 'validate',
-            maxConcurrentTasks: 99,
-            failedTasksTolerance: 99,
+            maxConcurrentTasks,
+            failedTasksTolerance,
             logVerbose,
         };
         await GenericTaskRunner.RunTasks<ICfnTask>(tasks, delegate);

src/cfn-binder/cfn-validate-task-provider.ts

@@ -1,4 +1,3 @@
-import { CloudFormation } from 'aws-sdk';
 import { ValidateTemplateInput } from 'aws-sdk/clients/cloudformation';
 import uuid = require('uuid');
 import { OrgFormationError } from '../org-formation-error';
@@ -7,6 +6,7 @@ import { ICfnTask } from './cfn-task-provider';
 import { CfnExpressionResolver } from '~core/cfn-expression-resolver';
 import { TemplateRoot } from '~parser/parser';
 import { PersistedState } from '~state/persisted-state';
+import { AwsUtil, CfnUtil } from '~util/aws-util';
 
 export class CfnValidateTaskProvider {
     constructor(private readonly template: TemplateRoot, private readonly state: PersistedState, private readonly logVerbose: boolean) {
@@ -46,12 +46,18 @@ export class CfnValidateTaskProvider {
             isDependency: (): boolean => false,
             action: 'Validate',
             perform: async (): Promise<void> => {
+                const customRoleName = await expressionResolver.resolveSingleExpression(binding.customRoleName, 'CustomRoleName');
+
                 const templateBody = await binding.template.createTemplateBodyAndResolve(expressionResolver);
+                const cfn = await AwsUtil.GetCloudFormation(binding.accountId, binding.region, customRoleName);
+
                 const validateInput: ValidateTemplateInput =  {
                     TemplateBody: templateBody,
                 };
 
-                const cfn = new CloudFormation({region: binding.region});
+                await CfnUtil.UploadTemplateToS3IfTooLarge(validateInput, binding, stackName, this.template.hash);
+
+
                 const result = await cfn.validateTemplate(validateInput).promise();
                 const missingParameters: string[] = [];
                 for (const param of result.Parameters) {

src/commands/init-organization-pipeline.ts

@@ -5,7 +5,7 @@ import { CreateStackInput, UpdateStackInput } from 'aws-sdk/clients/cloudformati
 import { PutObjectRequest } from 'aws-sdk/clients/s3';
 import { Command } from 'commander';
 import { WritableStream } from 'memory-streams';
-import { AwsUtil, DEFAULT_ROLE_FOR_CROSS_ACCOUNT_ACCESS } from '../util/aws-util';
+import { AwsUtil, CfnUtil, DEFAULT_ROLE_FOR_CROSS_ACCOUNT_ACCESS } from '../util/aws-util';
 import { ConsoleUtil } from '../util/console-util';
 import { OrgFormationError } from '../org-formation-error';
 import { BaseCliCommand, ICommandArgs } from './base-command';
@@ -122,31 +122,7 @@ export class InitPipelineCommand extends BaseCliCommand<IInitPipelineCommandArgs
             ],
         };
 
-        try {
-            await cfn.updateStack(stackInput).promise();
-            await cfn.waitFor('stackUpdateComplete', { StackName: stackName, $waiter: { delay: 1, maxAttempts: 60 * 30 } }).promise();
-        } catch (err) {
-            if (err && err.code === 'ValidationError' && err.message) {
-                const message = err.message as string;
-                if (-1 !== message.indexOf('ROLLBACK_COMPLETE')) {
-                    await cfn.deleteStack({ StackName: stackName }).promise();
-                    await cfn.waitFor('stackDeleteComplete', { StackName: stackName, $waiter: { delay: 1 } }).promise();
-                    await cfn.createStack(stackInput).promise();
-                    await cfn.waitFor('stackCreateComplete', { StackName: stackName, $waiter: { delay: 1, maxAttempts: 60 * 30 } }).promise();
-                } else if (-1 !== message.indexOf('does not exist')) {
-                    await cfn.createStack(stackInput).promise();
-                    await cfn.waitFor('stackCreateComplete', { StackName: stackName, $waiter: { delay: 1, maxAttempts: 60 * 30 } }).promise();
-                } else if (-1 !== message.indexOf('No updates are to be performed.')) {
-                    // ignore;
-                } else if (err.code === 'ResourceNotReady') {
-                    ConsoleUtil.LogError('error when executing cloudformation');
-                } else {
-                    throw err;
-                }
-            } else {
-                throw err;
-            }
-        }
+        await CfnUtil.UpdateOrCreateStack(cfn, stackInput);
     }
 
     private createTasksFile(path: string, stackName: string, resourcePrefix: string, stateBucketName: string, region: string, repositoryName: string): string {

src/commands/validate-organization.ts

@@ -0,0 +1,50 @@
+import { Command } from 'commander';
+import { ConsoleUtil } from '../util/console-util';
+import { BaseCliCommand } from './base-command';
+import { IUpdateOrganizationCommandArgs } from './update-organization';
+import { TemplateRoot } from '~parser/parser';
+import { GlobalState } from '~util/global-state';
+
+
+const commandName = 'validate <templateFile>';
+const commandDescription = 'validate organization resources';
+
+export class ValidateOrganizationCommand extends BaseCliCommand<IUpdateOrganizationCommandArgs> {
+
+    static SkipValidationForTasks = false;
+
+    constructor(command?: Command) {
+        super(command, commandName, commandDescription, 'templateFile');
+    }
+
+    public static async Perform(command: IUpdateOrganizationCommandArgs): Promise<void> {
+        const x = new ValidateOrganizationCommand();
+        await x.performCommand(command);
+    }
+
+    protected async performCommand(command: IUpdateOrganizationCommandArgs): Promise<void> {
+        const template = TemplateRoot.create(command.templateFile);
+        const state = await this.getState(command);
+        const templateHash = template.hash;
+
+        GlobalState.Init(state, template);
+
+        const lastHash = state.getTemplateHash();
+        if (command.forceDeploy === true) {
+            ConsoleUtil.LogInfo('organization validation forced.');
+        } else if (lastHash === templateHash) {
+            ConsoleUtil.LogInfo('organization up to date, no work to be done.');
+            return;
+        }
+
+        const binder = await this.getOrganizationBinder(template, state);
+        const tasks = binder.enumBuildTasks();
+        const createTasks = tasks.filter(x=>x.action === 'Create');
+        if (createTasks.length > 0) {
+            ConsoleUtil.LogWarning('Accounts where added to the organization model.');
+            ConsoleUtil.LogWarning('Tasks might depend on updating the organization.');
+            ConsoleUtil.LogWarning('validation of tasks will be skipped.');
+            ValidateOrganizationCommand.SkipValidationForTasks = true;
+        }
+    }
+}

src/commands/validate-stacks.ts

@@ -2,10 +2,12 @@ import { Command } from 'commander';
 import { ConsoleUtil } from '../util/console-util';
 import { BaseCliCommand } from './base-command';
 import { IUpdateStacksCommandArgs, UpdateStacksCommand } from './update-stacks';
+import { ValidateOrganizationCommand } from './validate-organization';
 import { CloudFormationBinder } from '~cfn-binder/cfn-binder';
 import { CfnTaskRunner } from '~cfn-binder/cfn-task-runner';
 import { CfnValidateTaskProvider } from '~cfn-binder/cfn-validate-task-provider';
 import { GlobalState } from '~util/global-state';
+import { Validator } from '~parser/validator';
 
 const commandName = 'validate-stacks <templateFile>';
 const commandDescription = 'validates the CloudFormation templates that will be generated';
@@ -28,6 +30,16 @@ export class ValidateStacksCommand extends BaseCliCommand<IUpdateStacksCommandAr
 
     public async performCommand(command: IUpdateStacksCommandArgs): Promise<void> {
         const templateFile = command.templateFile;
+
+        if (ValidateOrganizationCommand.SkipValidationForTasks) {
+            return;
+        }
+
+        Validator.validatePositiveInteger(command.maxConcurrentStacks, 'maxConcurrentStacks');
+        Validator.validatePositiveInteger(command.failedStacksTolerance, 'failedStacksTolerance');
+        Validator.validateBoolean(command.terminationProtection, 'terminationProtection');
+        Validator.validateBoolean(command.updateProtection, 'updateProtection');
+
         const template = UpdateStacksCommand.createTemplateUsingOverrides(command, templateFile);
         const state = await this.getState(command);
         GlobalState.Init(state, template);
@@ -40,6 +52,6 @@ export class ValidateStacksCommand extends BaseCliCommand<IUpdateStacksCommandAr
 
         const validationTaskProvider = new CfnValidateTaskProvider(template, state, command.verbose === true);
         const tasks = await validationTaskProvider.enumTasks(bindings);
-        await CfnTaskRunner.ValidateTemplates(tasks, command.verbose === true);
+        await CfnTaskRunner.ValidateTemplates(tasks, command.verbose === true, command.maxConcurrentStacks, command.failedStacksTolerance);
     }
 }