[WIP] Decreasing the false positive ratio with new pre-filters

Dockerfile

@@ -16,11 +16,12 @@ ENV LANG C.UTF-8
 ENV LANGUAGE C.UTF-8
 ENV LC_ALL C.UTF-8
 
-# Setting up terminal preferences
+# Add files into the container
 ADD ./ /opt/repo-supervisor
 
 # Install node version manager
-RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.2/install.sh | bash
+RUN /bin/bash -c "touch ~/.bashrc"
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.36.0/install.sh | bash
 RUN /bin/bash -c "source ~/.bashrc && nvm install 10"
 
 # Build scripts

config/filters/entropy.meter.json

@@ -4,6 +4,7 @@
   },
   "preFilters": [
     "whitelist.js",
+    "non.printable.characters.js",
     "authentication.urls.js",
     "css.selectors.js",
     "email.addresses.js",
@@ -12,19 +13,36 @@
     "multiple.words.js",
     "object.keys.identifiers.js",
     "skip.prefixes.js",
-    "dictionary.words.js"
+    "dictionary.words.js",
+    "regex.js"
   ],
   "options": {
     "entropyPrecision": 4,
     "maxAllowedEntropy": 4,
     "preFilters": {
+      "dictionaryWords": {
+        "requiredDictionaryWordsPercent": 0.3
+      },
       "skipPrefixes": [
-        "test-", "foobar-"
+        "test-",
+        "foobar-"
       ],
       "whitelist": [
-        "test", "foobar", "foo", "bar"
+        "test",
+        "foobar",
+        "foo",
+        "bar",
+        "abcdefghijklmnopqrstuvwxyz",
+        "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ",
+        "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "123456789abcdefghjkmnpqrstuvwxyz",
+        "0123456789abcdefghijklmnopqrstuvwxyz",
+        "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ",
+        "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_",
+        "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
       ],
       "minStringLength": 15
     }
   }
-}
+}

config/main.json

@@ -16,7 +16,7 @@
     }
   },
   "cli": {
-    "excludedPaths": ["^test", "^[.]git$"],
+    "excludedPaths": ["^test", "^[.]git$", "^dist", "^package.json$", "^package-lock.json$"],
     "messages": {
       "invalidArguments": "Invalid number of arguments.",
       "invalidDirectory": "\"%s\" is not a valid directory."
@@ -36,7 +36,7 @@
     "jwtTokenNotProvided": "JWT_SECRET is not set",
     "invalidReportID": "Report not found, invalid ID"
   },
-  "runTriggers": true,
+  "runTriggers": false,
   "triggerTextFormat": "mrkdwn",
   "triggerMessages": {
     "falsePositiveReported": ":warning: False positive reported (%s): %s",

src/filters/entropy.meter/pre.filters/dictionary.words.js

@@ -1,12 +1,11 @@
 'use strict';
 
 const dictionary = new Set(require('an-array-of-english-words'));
-
-const requiredPercent = 0.35;
+const config = require('./../../../../config/filters/entropy.meter.json');
 
 function splitIntoWords(str) {
   // Hyphens, underscores, commas, stops and numbers
-  str = str.replace(/[-_,.]/g, ' ')
+  str = str.replace(/[-_,.|()<>%]/g, ' ')
     .replace(/([0-9]+)/g, ' $1 ');
 
   // camelCase
@@ -37,5 +36,5 @@ module.exports = function filterCamelCaseWord(str) {
   );
 
   const percent = wordCount / words.length;
-  return percent < requiredPercent;
+  return percent < config.options.preFilters.dictionaryWords.requiredDictionaryWordsPercent;
 };

src/filters/entropy.meter/pre.filters/non.printable.characters.js

@@ -0,0 +1,10 @@
+/**
+ * Skip all strings containing non ASCII characters.
+ *
+ * Space " " (0x20) character is the first printable one in the ASCII table.
+ * The last printable one is tilde "~" (0x7E). This is the reason for defined
+ * range in the regular expression. We do not take UTF characters into account
+ * as those are not going to be a part of secret in vast majority of cases.
+ * It allows to reduce false positives significantly.
+ */
+module.exports = s => !s.match(/[^ -~]+/);

src/filters/entropy.meter/pre.filters/regex.js

@@ -0,0 +1,4 @@
+/**
+ * Skip all potential regular expressions.
+ */
+module.exports = s => !s.match(/\[a-z\]|\[0-9\]|(\(\?:[^]+\))|\[a-z0-9\]|\[a-z0-9_-\]|\[0-9A-Fa-f\]|\[a-fA-F0-9\]/i);

test/fixtures/unit/src/filters/entropy.meter/pre.filters/dictionary.words.json

@@ -30,7 +30,7 @@
     "Llxr8IOO5UucEqB8fng",
     "RNf_4j3t5ki2Qj-wnWjsNg",
     "RNf_4jdttki2Qj-wnwjsng",
-    "almost-ryeal-byet-noeut-there-jyet",
+    "almostt-ryeal-byet-noeut-there-jyet",
     "c6790fe8ae230ecce7fa521079e4ed08"
   ]
 }

test/fixtures/unit/src/filters/entropy.meter/pre.filters/non.printable.characters.json

@@ -0,0 +1,14 @@
+{
+  "printableASCIIcharacters": [
+    "0161ee9477df7286674533cd75b3a3f969b53be706cc046aed8fe1bcb3ba0ace",
+    "[fY28j7'Vb,(sWALk8NKH:ut^=N>rpP",
+    "9jFgE7<[[R]_~uVB!V3YC_#*994Vjm#,p5Bmz>%8Tm/.r#/ye!X$B7,A9?6{N=[eemCr",
+    "3NTtSeDPjnkx3jxrcN7vTQ9fj3dZjxJNWYDefZKfkvQ3"
+  ],
+  "nonPrintableASCIIcharacters": [
+    "0161ee9477⚠️df7286674533cd75b3a3f969b53be706cc046aed8fe1bcb3ba0ace",
+    "[fY28j7'Vb,(sWALk🐛8NKH:ut^=N>rpP",
+    "9jFgE7<[[R]_🐞~uVB!V3YC_#*994Vjm#,p5Bmz>%8Tm/.r#/ye!X$B7,A9?6{N=[eemCr",
+    "3NTtSeDPjnkx3jxrcN7vTQ9fj3文dZjxJNWYDefZKfkvQ3"
+  ]
+}

test/fixtures/unit/src/filters/entropy.meter/pre.filters/regex.json

@@ -0,0 +1,15 @@
+{
+  "commonRegexPattern": [
+    "[a-z]+",
+    "[0-9]{1,2}",
+    "^test-[a-z]+$",
+    "[a-z0-9_-]",
+    "[0-9A-Fa-f]",
+    "[a-fA-F0-9]"
+  ],
+  "nonStandardRegexPattern": [
+    "[a-c]+",
+    "[1234]*",
+    "^.*$"
+  ]
+}

test/unit/src/filters/entropy.meter/pre.filters/non.printable.characters.js

@@ -0,0 +1,16 @@
+const filter = require(`${global.srcPath}/filters/entropy.meter/pre.filters/non.printable.characters.js`);
+const fixtures = global.getFixtures(__filename);
+
+describe('Pre filter -> Non-printable characters', () => {
+  it('should skip all non ASCII printable characters', () => {
+    fixtures.nonPrintableASCIIcharacters.forEach((words) => {
+      expect(filter(words), words).to.be.equal(false);
+    });
+  });
+
+  it('should allow all non ASCII printable characters', () => {
+    fixtures.printableASCIIcharacters.forEach((words) => {
+      expect(filter(words), words).to.be.equal(true);
+    });
+  });
+});

test/unit/src/filters/entropy.meter/pre.filters/regex.js

@@ -0,0 +1,16 @@
+const filter = require(`${global.srcPath}/filters/entropy.meter/pre.filters/regex.js`);
+const fixtures = global.getFixtures(__filename);
+
+describe('Pre filter -> Regular expressions', () => {
+  it('should ignore all defined regular expressions', () => {
+    fixtures.commonRegexPattern.forEach((pattern) => {
+      expect(filter(pattern), pattern).to.be.equal(false);
+    });
+  });
+
+  it('should not ignore all non-standard patterns in regular expressions', () => {
+    fixtures.nonStandardRegexPattern.forEach((pattern) => {
+      expect(filter(pattern), pattern).to.be.equal(true);
+    });
+  });
+});