Merge pull request #1138 from HubSpot/br/scheduled-cli-tests

test: Adding scheduled actions to test latest CLI release

.github/workflows/build.yml

@@ -1,9 +1,7 @@
 on: [push]
-
 jobs:
   build:
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js
@@ -46,7 +44,7 @@ jobs:
           status: ${{ job.status }}
           notify_when: 'failure'
           notification_title: 'CLI build failure'
-          message_format: '{emoji} *Build* {status_message} in <{repo_url}|{repo}> on <{commit_url}|{commit_sha}>"'
+          message_format: '{emoji} *Build* {status_message} in <{repo_url}|{repo}> on <{commit_url}|{commit_sha}>'
           footer: '<{run_url}|View Run>'
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

.github/workflows/testLatestPublish.yml

@@ -0,0 +1,38 @@
+on:
+  push:
+    tags:
+      - '*'
+  # Scheduled jobs will only run on the default branch
+  schedule:
+    - cron: '* */5 * * *' # Run every 5 hours
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      # Sleep when triggered by new tag to wait for npm publish
+      - name: Sleep For 3 Mins
+        run: sleep 180s
+        shell: bash
+      - name: Use Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: 16.x
+      - name: Install Deps
+        run: npm install
+      - name: Test Latest CLI Release
+        run: |
+          npm run test-cli --cliVersion="5.0.2"
+        env:
+          PORTAL_ID: ${{ secrets.ACCEPTANCE_TEST_PORTAL_ID }}
+          PERSONAL_ACCESS_KEY: ${{ secrets.ACCEPTANCE_TEST_PERSONAL_ACCESS_KEY }}
+      - name: Release Failure Slack Report
+        uses: ravsamhq/notify-slack-action@v2
+        if: ${{ always() && github.ref_name == 'main' }}
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+          notification_title: 'CLI latest release failure'
+          message_format: '{emoji} *Release* {status_message}'
+          footer: '<{run_url}|View Run> | <https://www.npmjs.com/package/@hubspot/cli?activeTab=versions|View in NPM>'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

acceptance-tests/lib/env.js

@@ -34,6 +34,7 @@ const getEnvValue = envVariable => {
 const setArgsOverrides = args => {
   args.portalId && (argsOverrides.portalId = args.portalId);
   args.cliPath && (argsOverrides.cliPath = args.cliPath);
+  args.cliVersion && (argsOverrides.cliVersion = args.cliVersion);
   args.personalAccessKey &&
     (argsOverrides.personalAccessKey = args.personalAccessKey);
   argsOverrides.qa = args.qa;
@@ -45,6 +46,7 @@ const envOverrides = getTruthyValuesOnly({
   portalId: getEnvValue('PORTAL_ID') || getEnvValue('ACCOUNT_ID'),
   cliPath: getEnvValue('CLI_PATH'),
   personalAccessKey: getEnvValue('PERSONAL_ACCESS_KEY'),
+  cliVersion: getEnvValue('CLI_VERSION'),
 });
 
 const getTestConfig = () => {
@@ -57,7 +59,13 @@ const getTestConfig = () => {
     );
   }
 
-  if (!config.cliPath) {
+  if (config.cliPath && config.cliVersion) {
+    throw new Error(
+      'You cannot specify both a cliPath and a cliVersion. Remove one and try again.'
+    );
+  }
+
+  if (!config.cliPath && !config.cliVersion) {
     const defaultPath = path.join(process.cwd(), DEFAULT_CLI_PATH);
 
     if (existsSync(defaultPath)) {

acceptance-tests/run-tests

@@ -18,12 +18,16 @@ setArgsOverrides(
     .option('accountId', {
       alias: ['p', 'portalId', 'a'],
       type: 'string',
-      description: 'Account ID',
+      description: 'The account ID to use',
     })
     .option('cliPath', {
       alias: 'c',
       type: 'string',
-      description: 'CLI path',
+      description: 'The path to the CLI',
+    })
+    .option('cliVersion', {
+      type: "string",
+      description: "The npm version to use. The test will use npx to download and run the CLI."
     })
     .option('personalAccessKey', {
       alias: 'pak',
@@ -33,7 +37,7 @@ setArgsOverrides(
     .option('qa', {
       type: 'boolean',
       default: false,
-      description: 'Set this if you are using a app.hubspotqa.com site',
+      description: 'Set this if you are using an app.hubspotqa.com account',
     })
     .option('headless', {
       type: 'boolean',
@@ -55,7 +59,7 @@ setArgsOverrides(
     testRunner.addReporter(reporter);
 
     global.config = getTestConfig();
-    global.cli = cmd.createCli(global.config.cliPath);
+    global.cli = cmd.createCli(global.config.cliPath, global.config.cliVersion);
 
     await global.cli.execute(
       ['init', `--c="${CONFIG_FILE_NAME}"`],

acceptance-tests/tests/helpers/cmd.js

@@ -12,21 +12,31 @@ const PATH = process.env.PATH;
 
 /**
  * Creates a child process with script path
- * @param {string} processPath Path of the process to execute
+ * @param {string} cliPath Path of the CLI process to execute
+ * @param {string} cliVersion NPM Version number
  * @param {Array} args Arguments to the command
  * @param {Object} env (optional) Environment variables
  */
-function createProcess(processPath, args = [], env = null) {
-  // Ensure that path exists
-  if (!processPath || !existsSync(processPath)) {
-    throw new Error('Invalid process path');
-  }
+function createProcess(cliPath, cliVersion, args = [], env = null) {
+  let processCommand;
 
-  args = [processPath].concat(args);
+  if (cliVersion) {
+    processCommand = 'npx';
+    args = ['--yes', '--package', `@hubspot/cli@${cliVersion}`, 'hs'].concat(
+      args
+    );
+  } else {
+    // Ensure that path exists
+    if (!cliPath || !existsSync(cliPath)) {
+      throw new Error(`Invalid process path ${cliPath}`);
+    }
+    processCommand = 'node';
+    args = [cliPath].concat(args);
+  }
 
   // This works for node based CLIs, but can easily be adjusted to
   // any other process installed in the system
-  return spawn('node', args, {
+  return spawn(processCommand, args, {
     env: Object.assign(
       {
         NODE_ENV: 'test',
@@ -48,7 +58,13 @@ function createProcess(processPath, args = [], env = null) {
  * @param {Array} inputs (Optional) Array of inputs (user responses)
  * @param {Object} opts (optional) Environment variables
  */
-function executeWithInput(processPath, args = [], inputs = [], opts = {}) {
+function executeWithInput(
+  cliPath,
+  cliVersion,
+  args = [],
+  inputs = [],
+  opts = {}
+) {
   if (!Array.isArray(inputs)) {
     opts = inputs;
     inputs = [];
@@ -59,7 +75,7 @@ function executeWithInput(processPath, args = [], inputs = [], opts = {}) {
   }
 
   const { env = opts.env, timeout = 500, maxTimeout = 10000 } = opts;
-  const childProcess = createProcess(processPath, args, env);
+  const childProcess = createProcess(cliPath, cliVersion, args, env);
   childProcess.stdin.setEncoding('utf-8');
 
   let currentInputTimeout, killIOTimeout;
@@ -171,8 +187,8 @@ function executeWithInput(processPath, args = [], inputs = [], opts = {}) {
 
 module.exports = {
   createProcess,
-  createCli: processPath => ({
-    execute: (...args) => executeWithInput(processPath, ...args),
+  createCli: (cliPath, cliVersion) => ({
+    execute: (...args) => executeWithInput(cliPath, cliVersion, ...args),
   }),
   DOWN: '\x1B\x5B\x42',
   UP: '\x1B\x5B\x41',