Improve template CI tests (close vuejs-templates#1218)

* Batman! * fix wrong job name * fix working dir paths * fix node_modules path * test other step order * try other shell * ...and another one * rewrite test.sh * try running a bash script instead * and another attempt. * make executable * anpother try... * fix workflow errors * fix path * Batman! * added scenarios and made it work!! * refactors and comments * just fumbling around * adfda * adjust test scenarios for autoInstall * x<vb<xb * first try since a long time * small fix * fix path to vue-cli * fix scenarios * fix persist paths * testing with quotes for the path * removing unhelpful quotes * New Test * fixed indentation * test with full setup * 2nd full test * bump jest version * other docker image * test with runinband * test with airbnb/karma as well * remove superfluous checkout * run unit and e2e individually. * fix eslint paths for test subdirs * adjust for airbnb preset * only lint e2e specs, config differences are unfixable. * adding a small readme * remove test.sh not needed anymore
xu455255849 · Jan 13, 2018 · fd65d24 · fd65d24
1 parent f21376d
commit fd65d24
Show file tree

Hide file tree

Showing 11 changed files with 247 additions and 27 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,137 @@
+version: 2
+vm_settings: &vm_settings
+  docker:
+    - image: circleci/node:8.9.4-browsers
+
+jobs:
+  install_template_deps:
+    <<: *vm_settings
+    working_directory: ~/project/webpack-template
+    steps:
+      - checkout
+      - restore_cache:
+          key: template-cache-{{ checksum "package.json" }}
+      - run:
+          name: Install npm dependencies
+          command: npm install
+      - save_cache:
+          key: template-cache-{{ checksum "package.json" }}
+          paths:
+            - node_modules
+      - run:
+          name: Rollout minimal scenario
+          command: VUE_TEMPL_TEST=minimal node_modules/.bin/vue init . test-minimal
+      - run:
+          name: Rollout full scenario
+          command: VUE_TEMPL_TEST=full node_modules/.bin/vue init . test-full
+      - run:
+          name: Rollout full-karma-airbnb scenario
+          command: VUE_TEMPL_TEST=full-karma-airbnb node_modules/.bin/vue init . test-full-karma-airbnb
+      - persist_to_workspace:
+          root: ~/project/webpack-template
+          paths:
+            - node_modules
+            - test-*
+
+  scenario_minimal:
+    <<: *vm_settings
+    environment:
+      - VUE_TEMPL_TEST: minimal
+    working_directory: ~/project/webpack-template/test-minimal
+    steps:
+      - attach_workspace:
+          at: '~/project/webpack-template'
+      - restore_cache:
+          key: template-cache-minimal-{{ checksum "package.json" }}
+      - run:
+          name: Install npm dependencies
+          command: npm install
+      - save_cache:
+          key: template-cache-minimal-{{ checksum "package.json" }}
+          paths:
+            - node_modules
+      - run:
+          name: Test build
+          command: npm run build
+
+  scenario_full:
+    <<: *vm_settings
+    working_directory: ~/project/webpack-template/test-full
+    environment:
+      - VUE_TEMPL_TEST: full
+    steps:
+      - attach_workspace:
+          at: '~/project/webpack-template'
+      - restore_cache:
+          key: template-cache-full-{{ checksum "package.json" }}
+      - run:
+          name: Install npm dependencies
+          command: npm install
+      - save_cache:
+          key: template-cache-full-{{ checksum "package.json" }}
+          paths:
+            - node_modules
+      - run:
+          name: Run Lint
+          command: npm run lint -- --fix
+      - run:
+          name: Run Unit tests
+          command: npm run unit
+          when: always
+      - run:
+          name: Run e2e tests
+          command: npm run e2e
+          when: always
+      - run:
+          name: Test build
+          command: npm run build
+          when: always
+
+  scenario_full-karma-airbnb:
+    <<: *vm_settings
+    working_directory: ~/project/webpack-template/test-full-karma-airbnb
+    environment:
+      - VUE_TEMPL_TEST: full-karma-airbnb
+    steps:
+      - attach_workspace:
+          at: '~/project/webpack-template'
+      - restore_cache:
+          key: template-cache-full-karma-airbnb-{{ checksum "package.json" }}
+      - run:
+          name: Install npm dependencies
+          command: npm install
+      - save_cache:
+          key: template-cache-full-karma-airbnb-{{ checksum "package.json" }}
+          paths:
+            - node_modules
+      - run:
+          name: Run Lint
+          command: npm run lint -- --fix
+      - run:
+          name: Run Unit tests
+          command: npm run unit
+          when: always
+      - run:
+          name: Run e2e tests
+          command: npm run e2e
+          when: always
+      - run:
+          name: Test build
+          command: npm run build
+          when: always
+
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - install_template_deps
+      - scenario_minimal:
+          requires:
+            - install_template_deps
+      - scenario_full:
+          requires:
+            - install_template_deps
+      - scenario_full-karma-airbnb:
+          requires:
+            - install_template_deps
diff --git a/circle.yml b/circle.yml
diff --git a/meta.js b/meta.js
@@ -1,5 +1,6 @@
 const path = require('path')
 const fs = require('fs')
+
 const {
   sortDependencies,
   installDependencies,
@@ -10,9 +11,16 @@ const pkg = require('./package.json')
 
 const templateVersion = pkg.version
 
+const { addTestAnswers } = require('./scenarios')
+
 module.exports = {
+  metalsmith: {
+    // When running tests for the template, this adds answers for the selected scenario
+    before: addTestAnswers
+  },
   helpers: {
     if_or(v1, v2, options) {
+
       if (v1 || v2) {
         return options.fn(this)
       }
@@ -26,21 +34,25 @@ module.exports = {
 
   prompts: {
     name: {
+      when: 'isNotTest',
       type: 'string',
       required: true,
       message: 'Project name',
     },
     description: {
+      when: 'isNotTest',
       type: 'string',
       required: false,
       message: 'Project description',
       default: 'A Vue.js project',
     },
     author: {
+      when: 'isNotTest',
       type: 'string',
       message: 'Author',
     },
     build: {
+      when: 'isNotTest',
       type: 'list',
       message: 'Vue build',
       choices: [
@@ -58,15 +70,17 @@ module.exports = {
       ],
     },
     router: {
+      when: 'isNotTest',
       type: 'confirm',
       message: 'Install vue-router?',
     },
     lint: {
+      when: 'isNotTest',
       type: 'confirm',
       message: 'Use ESLint to lint your code?',
     },
     lintConfig: {
-      when: 'lint',
+      when: 'isNotTest && lint',
       type: 'list',
       message: 'Pick an ESLint preset',
       choices: [
@@ -88,11 +102,12 @@ module.exports = {
       ],
     },
     unit: {
+      when: 'isNotTest',
       type: 'confirm',
       message: 'Set up unit tests',
     },
     runner: {
-      when: 'unit',
+      when: 'isNotTest && unit',
       type: 'list',
       message: 'Pick a test runner',
       choices: [
@@ -114,10 +129,12 @@ module.exports = {
       ],
     },
     e2e: {
+      when: 'isNotTest',
       type: 'confirm',
       message: 'Setup e2e tests with Nightwatch?',
     },
     autoInstall: {
+      when: 'isNotTest',
       type: 'list',
       message:
         'Should we run `npm install` for you after the project has been created? (recommended)',

diff --git a/scenarios/README.md b/scenarios/README.md
@@ -0,0 +1,27 @@
+## What is this folder?
+
+This folder contains test scenarios for the automated tests of the template on CircleCI.
+
+Each `.json`file contains an object that represents a set of answers to the questions that vue-cli asks the user when installing the template.
+
+With the code from `index.js`, we insert those answers into the metalsmith metadata and skip the inquirer questions, thus allowing us to run different test scenarios in CI without having to actually provide any answers to inquirer or to mock it.
+
+## The scenarios
+
+We currently have 3 scenrios set up:
+
+1. 'minimal': it basically answers "no" to ever choice, so no router, no elint, no tests
+2. 'full': It answers "yes" to every choice. With router, with linting (standard), with full tests (jest & e2e)
+3. 'full-airbnb-karma': like 'full', but using airbnb eslint config instead od standard and karma instead of jest for unnit tests.
+
+Other permutations might be worth testing to secure against edge cases, but this gives us a decent level of security over common combinations.
+
+## How to use it?
+
+Choosing a scenario is done through setting an ENV variable named `VUE_TEMPL_TEST`.
+
+You can run a scenario yourself by running this in your terminal:
+
+````
+VUE_TEMPL_TEST=minimal vue init webpack your-directory
+```
diff --git a/scenarios/full-karma-airbnb.json b/scenarios/full-karma-airbnb.json
@@ -0,0 +1,14 @@
+{ 
+  "noEscape": true,
+  "name": "test",
+  "description": "A Vue.js project",
+  "author": "CircleCI",
+  "build": "standalone",
+  "router": false,
+  "lint": true,
+  "lintConfig": "airbnb",
+  "unit": true,
+  "runner": "karma",
+  "e2e": true,
+  "autoInstall": false
+}
diff --git a/scenarios/full.json b/scenarios/full.json
@@ -0,0 +1,14 @@
+{ 
+  "noEscape": true,
+  "name": "test",
+  "description": "A Vue.js project",
+  "author": "CircleCI",
+  "build": "runtime",
+  "router": false,
+  "lint": true,
+  "lintConfig": "standard",
+  "unit": true,
+  "runner": "jest",
+  "e2e": true,
+  "autoInstall": false
+}
diff --git a/scenarios/index.js b/scenarios/index.js
@@ -0,0 +1,19 @@
+const scenarios = [
+  'full', 
+  'full-karma-airbnb', 
+  'minimal'
+]
+
+const index = scenarios.indexOf(process.env.VUE_TEMPL_TEST)
+
+const isTest = exports.isTest = index !== -1
+
+const scenario = isTest && require(`./${scenarios[index]}.json`)
+
+exports.addTestAnswers = (metalsmith, options, helpers) => {
+  Object.assign(metalsmith.metadata(), {
+    isNotTest: !isTest,
+    ...(isTest ? scenario : undefined)
+  })
+  // console.log(metalsmith.metadata())
+}
diff --git a/scenarios/minimal.json b/scenarios/minimal.json
@@ -0,0 +1,14 @@
+{ 
+  "noEscape": true,
+  "name": "test-minimal",
+  "description": "Testing the minimal template setup",
+  "author": "CircleCI",
+  "build": "standalone",
+  "router": false,
+  "lint": false,
+  "lintConfig": "standard",
+  "unit": false,
+  "runner": "jest",
+  "e2e": false,
+  "autoInstall": false
+}
diff --git a/template/package.json b/template/package.json
@@ -20,7 +20,7 @@
     "test": "{{#unit}}npm run unit{{/unit}}{{#unit}}{{#e2e}} && {{/e2e}}{{/unit}}{{#e2e}}npm run e2e{{/e2e}}",
     {{/if_or}}
     {{#lint}}
-    "lint": "eslint --ext .js,.vue src{{#unit}} test/unit/specs{{/unit}}{{#e2e}} test/e2e/specs{{/e2e}}",
+    "lint": "eslint --ext .js,.vue src{{#unit}} test/unit{{/unit}}{{#e2e}} test/e2e/specs{{/e2e}}",
     {{/lint}}
     "build": "node build/build.js"
   },
@@ -52,7 +52,7 @@
     "babel-jest": "^21.0.2",
     "babel-plugin-dynamic-import-node": "^1.2.0",
     "babel-plugin-transform-es2015-modules-commonjs": "^6.26.0",
-    "jest": "^21.2.0",
+    "jest": "^22.0.4",
     "jest-serializer-vue": "^0.3.0",
     "vue-jest": "^1.0.2",
     {{/if_eq}}

diff --git a/template/test/unit/karma.conf.js b/template/test/unit/karma.conf.js
@@ -5,7 +5,7 @@
 
 var webpackConfig = require('../../build/webpack.test.conf')
 
-module.exports = function (config) {
+module.exports = function karmaConfig (config) {
   config.set({
     // to run in additional browsers:
     // 1. install corresponding karma launcher

diff --git a/test.sh b/test.sh