Merge pull request #467 from DannyBen/add/invalid-schema-tests

Improve JSON schema tests

spec/fixtures/schemas_invalid/README.md

@@ -0,0 +1,6 @@
+This folder contains invalid schema validations.
+
+Each subfolder is named after the schema.
+
+The tests can be run with `run schema invalid` and are included in the
+`run schema all` test (which is executed in CI).

spec/fixtures/schemas_invalid/bashly/1.yml

@@ -0,0 +1 @@
+invalid: not allowed

spec/fixtures/schemas_invalid/bashly/2.yml

@@ -0,0 +1,4 @@
+args:
+- name: path
+  x_valid: allowed
+  invalid: not allowed

spec/fixtures/schemas_invalid/bashly/3.yml

@@ -0,0 +1,4 @@
+flags:
+- long: --path
+  x_valid: allowed
+  invalid: not allowed

spec/fixtures/schemas_invalid/bashly/4.yml

@@ -0,0 +1,4 @@
+environment_variables:
+- name: api_key
+  x_valid: allowed
+  invalid: not allowed

spec/fixtures/schemas_invalid/settings/1.yml

@@ -0,0 +1 @@
+foo: bar

spec/fixtures/schemas_invalid/settings/2.yml

@@ -0,0 +1,2 @@
+usage_colors: true
+partials_extension: [1, 2]

spec/fixtures/schemas_invalid/strings/1.yml

@@ -0,0 +1 @@
+foo: bar

spec/fixtures/schemas_invalid/strings/2.yml

@@ -0,0 +1 @@
+usage: true

support/runfile/schema.runfile

@@ -6,6 +6,24 @@ action :all do
   check_settings
   check_strings
   check_arbitrary
+  check_invalid
+end
+
+help 'Test any file against any schema'
+usage 'file [--anti] PATH [SCHEMA]'
+option '-a, --anti', 'Run anticheck instead (expect failure)'
+param 'PATH', 'The path to the tested YAML file'
+param 'SCHEMA', 'The name of the schema (bashly, settings of strings) [default: bashly]'
+example 'run schema file -a spec/fixtures/schemas_invalid/bashly/1.yml'
+example 'run schema file -a spec/fixtures/schemas_invalid/settings/2.yml settings'
+action(:file) do |args|
+  schema = args['SCHEMA'] || 'bashly'
+  if args['--anti']
+    schema_anticheck args['PATH'], schema
+  else
+    schema_check args['PATH'], schema
+  end
+  say "\ngub`PASS`"
 end
 
 help 'Test the bashly schema against a single example'
@@ -24,31 +42,52 @@ action(:strings) { check_strings }
 help 'Test the bashly schema against a bashly configuration that includes arbitrary (x-) keys'
 action(:arbitrary) { check_arbitrary }
 
+help 'Verify that all the invalid schemas in spec/fixtures/schemas_invalid fail as expected'
+action(:invalid) { check_invalid }
+
 helpers do
   def check_examples
     say "\ngub`Examples`"
     Example.all.each do |example|
       file = example.yaml_path
       schema_check file
     end
+    say "\ngub`Examples PASS`"
   end
 
   def check_settings
-    say "\ngub`Settings`"
+    say "\ngub`Settings schema`"
     file = 'lib/bashly/libraries/settings/settings.yml'
     schema_check file, :settings
+    say "\ngub`Settings schema PASS`"
   end
 
   def check_strings
-    say "\ngub`Strings`"
+    say "\ngub`Strings schema`"
     file = 'lib/bashly/libraries/strings/strings.yml'
     schema_check file, :strings
+    say "\ngub`Strings schema PASS`"
   end
 
   def check_arbitrary
-    say "\ngub`Arbitrary`"
+    say "\ngub`Arbitrary arguments`"
     file = 'spec/fixtures/script/x_arbitrary.yml'
     schema_check file
+    say "\ngub`Arbitrary arguments PASS`"
+  end
+
+  def check_invalid
+    say "\ngub`Invalid files`"
+    basedir = 'spec/fixtures/schemas_invalid'
+    files = Dir.chdir(basedir) { Dir['**/*.yml'] }
+
+    files.each do |file|
+      schema = File.dirname file
+      path = "#{basedir}/#{file}"
+      schema_anticheck path, schema
+    end
+
+    say "\ngub`Invalid files PASS`"
   end
 
   def schema_check(file, schema = 'bashly')
@@ -58,4 +97,12 @@ helpers do
 
     abort 'Failed' unless success
   end
+
+  def schema_anticheck(file, schema = 'bashly')
+    command = "check-jsonschema --schemafile schemas/#{schema}.json #{file}"
+    say "\nnb`$ check-jsonschema` [m`#{schema}`] bb`#{file}`"
+    success = system "#{command}"
+
+    abort 'Failed' if success
+  end
 end