From 2e77117b7c6bb681786be0c8f36cbcd22b08e006 Mon Sep 17 00:00:00 2001 From: Prachit Date: Sat, 26 Oct 2024 20:12:59 +0530 Subject: [PATCH 1/9] feat(csv): add sensitive data check for .csv and .xlsx files --- .gitignore | 1 + package-lock.json | 111 ++++++++++++++- package.json | 2 + src/proxy/chain.js | 4 +- .../push-action/checkSensitiveData.js | 131 ++++++++++++++++++ src/proxy/processors/push-action/getDiff.js | 1 + src/proxy/processors/push-action/index.js | 1 + test/CheckSensitive.test.js | 59 ++++++++ test/CreateExcel.js | 24 ++++ test/chain.test.js | 9 ++ test/test_data/sensitive_data.csv | 3 + test/test_data/sensitive_data2.xlsx | Bin 0 -> 16196 bytes 12 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 src/proxy/processors/push-action/checkSensitiveData.js create mode 100644 test/CheckSensitive.test.js create mode 100644 test/CreateExcel.js create mode 100644 test/test_data/sensitive_data.csv create mode 100644 test/test_data/sensitive_data2.xlsx diff --git a/.gitignore b/.gitignore index 1849589c..b398dc87 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ yarn-debug.log* yarn-error.log* lerna-debug.log* package-lock.json +/git-proxy-test # Diagnostic reports (https://nodejs.org/api/report.html) diff --git a/package-lock.json b/package-lock.json index 51e24d1a..de2d0a73 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "concurrently": "^8.0.0", "connect-mongo": "^5.1.0", "cors": "^2.8.5", + "csv-parser": "^3.0.0", "diff2html": "^3.4.33", "express": "^4.18.2", "express-http-proxy": "^2.0.0", @@ -47,6 +48,7 @@ "react-html-parser": "^2.0.2", "react-router-dom": "6.26.2", "uuid": "^10.0.0", + "xlsx": "^0.18.5", "yargs": "^17.7.2" }, "bin": { @@ -3766,6 +3768,14 @@ "node": ">=4.0" } }, + "node_modules/adler-32": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", + "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -4534,6 +4544,18 @@ "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", "dev": true }, + "node_modules/cfb": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", + "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", + "dependencies": { + "adler-32": "~1.3.0", + "crc-32": "~1.2.0" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/chai": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", @@ -4867,6 +4889,14 @@ "node": ">=6" } }, + "node_modules/codepage": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", + "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -5218,6 +5248,17 @@ "typescript": ">=4" } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -5245,6 +5286,20 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-2.6.21.tgz", "integrity": "sha512-Z1PhmomIfypOpoMjRQB70jfvy/wxT50qW08YXO5lMIJkrdq4yOTR+AW7FqutScmB9NkLwxo+jU+kZLbofZZq/w==" }, + "node_modules/csv-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz", + "integrity": "sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==", + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "csv-parser": "bin/csv-parser" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/cypress": { "version": "13.14.2", "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.14.2.tgz", @@ -7241,6 +7296,14 @@ "node": ">= 0.6" } }, + "node_modules/frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", @@ -9689,7 +9752,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -11983,6 +12045,17 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/ssf": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", + "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", + "dependencies": { + "frac": "~1.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/sshpk": { "version": "1.18.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz", @@ -13076,6 +13149,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/wmf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", + "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/word": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", + "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/workerpool": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz", @@ -13217,6 +13306,26 @@ "typedarray-to-buffer": "^3.1.5" } }, + "node_modules/xlsx": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", + "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "dependencies": { + "adler-32": "~1.3.0", + "cfb": "~1.2.1", + "codepage": "~1.15.0", + "crc-32": "~1.2.1", + "ssf": "~0.11.2", + "wmf": "~1.0.1", + "word": "~0.3.0" + }, + "bin": { + "xlsx": "bin/xlsx.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index f4d6b467..473bd225 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "concurrently": "^8.0.0", "connect-mongo": "^5.1.0", "cors": "^2.8.5", + "csv-parser": "^3.0.0", "diff2html": "^3.4.33", "express": "^4.18.2", "express-http-proxy": "^2.0.0", @@ -73,6 +74,7 @@ "react-html-parser": "^2.0.2", "react-router-dom": "6.26.2", "uuid": "^10.0.0", + "xlsx": "^0.18.5", "yargs": "^17.7.2" }, "devDependencies": { diff --git a/src/proxy/chain.js b/src/proxy/chain.js index 11e6ae10..332e5379 100644 --- a/src/proxy/chain.js +++ b/src/proxy/chain.js @@ -9,12 +9,14 @@ const pushActionChain = [ proc.push.checkIfWaitingAuth, proc.push.pullRemote, proc.push.writePack, - proc.push.getDiff, + proc.push.getDiff, + proc.push.checkSensitiveData, //checkSensitiveData added proc.push.clearBareClone, proc.push.scanDiff, proc.push.blockForAuth, ]; + const pullActionChain = [proc.push.checkRepoInAuthorisedList]; let pluginsInserted = false; diff --git a/src/proxy/processors/push-action/checkSensitiveData.js b/src/proxy/processors/push-action/checkSensitiveData.js new file mode 100644 index 00000000..a33614e1 --- /dev/null +++ b/src/proxy/processors/push-action/checkSensitiveData.js @@ -0,0 +1,131 @@ +const fs = require('fs'); +const csv = require('csv-parser'); +const XLSX = require('xlsx'); +const path = require('path'); +const { exec: getDiffExec } = require('./getDiff'); + +// Function to check for sensitive data patterns +const checkForSensitiveData = (cell) => { + const sensitivePatterns = [ + /\d{3}-\d{2}-\d{4}/, // Social Security Number (SSN) + /\b\d{16}\b/, // Credit card numbers + /\b\d{5}-\d{4}\b/, // ZIP+4 codes + // Add more patterns as needed + ]; + return sensitivePatterns.some(pattern => { + if (pattern.test(String(cell))) { + console.log(`\x1b[31mDetected sensitive data: ${cell}\x1b[0m`); // Log the detected sensitive data in red + return true; + } + return false; + }); +}; + +// Function to process CSV files +const processCSV = async (filePath) => { + return new Promise((resolve, reject) => { + let sensitiveDataFound = false; + + fs.createReadStream(filePath) + .pipe(csv()) + .on('data', (row) => { + for (const [key, value] of Object.entries(row)) { + if (checkForSensitiveData(value)) { + console.log(`\x1b[33mSensitive data found in CSV: ${key}: ${value}\x1b[0m`); // Log in yellow + sensitiveDataFound = true; + } + } + }) + .on('end', () => { + if (!sensitiveDataFound) { + console.log('No sensitive data found in CSV.'); + } + resolve(sensitiveDataFound); // Resolve with the flag indicating if sensitive data was found + }) + .on('error', (err) => { + console.error(`Error reading CSV file: ${err.message}`); + reject(err); // Reject the promise on error + }); + }); +}; + +// Function to process XLSX files +const processXLSX = async (filePath) => { + return new Promise((resolve, reject) => { + let sensitiveDataFound = false; + + try { + const workbook = XLSX.readFile(filePath); + const sheetName = workbook.SheetNames[0]; + const sheet = workbook.Sheets[sheetName]; + const jsonData = XLSX.utils.sheet_to_json(sheet); + + jsonData.forEach((row) => { + for (const [key, value] of Object.entries(row)) { + if (checkForSensitiveData(value)) { + console.log(`\x1b[33mSensitive data found in XLSX: ${key}: ${value}\x1b[0m`); // Log in yellow + sensitiveDataFound = true; + } + } + }); + + if (!sensitiveDataFound) { + console.log('No sensitive data found in XLSX.'); + } + resolve(sensitiveDataFound); // Resolve with the flag indicating if sensitive data was found + } catch (error) { + console.error(`Error reading XLSX file: ${error.message}`); + reject(error); // Reject the promise on error + } + }); +}; + +// Function to parse the file based on its extension +const parseFile = async (filePath) => { + const ext = path.extname(filePath).toLowerCase(); + + switch (ext) { + case '.csv': + return await processCSV(filePath); + case '.xlsx': + return await processXLSX(filePath); + default: + console.log(`Unsupported file type: ${ext} for file: ${filePath}`); + return false; // Indicate that no sensitive data was found for unsupported types + } +}; + +// Async exec function to handle actions +const exec = async (req, action) => { + // getDiffExec(req, action); // Call to getDiffExec if necessary + + const diffStep = action.steps.find((s) => s.stepName === 'diff'); + + if (diffStep && diffStep.content) { + console.log('Diff content:', diffStep.content); + + const filePaths = diffStep.content.filePaths || []; + + if (filePaths.length > 0) { + // Check for sensitive data in all files + const sensitiveDataFound = await Promise.all(filePaths.map(parseFile)); + const anySensitiveDataDetected = sensitiveDataFound.some(found => found); // Check if any file reported sensitive data + + if (anySensitiveDataDetected) { + action.pushBlocked = true; // Block the push + action.error = true; // Set error flag + action.errorMessage = 'Your push has been blocked due to sensitive data detection.'; // Set error message + console.log(action.errorMessage); + } + } else { + console.log('No file paths provided in the diff step.'); + } + } else { + console.log('No diff content available.'); + } + + return action; // Returning action for testing purposes +}; + +exec.displayName = 'logFileChanges.exec'; +exports.exec = exec; diff --git a/src/proxy/processors/push-action/getDiff.js b/src/proxy/processors/push-action/getDiff.js index 4344810b..ed508ab9 100644 --- a/src/proxy/processors/push-action/getDiff.js +++ b/src/proxy/processors/push-action/getDiff.js @@ -1,6 +1,7 @@ const child = require('child_process'); const Step = require('../../actions').Step; + const exec = async (req, action) => { const step = new Step('diff'); diff --git a/src/proxy/processors/push-action/index.js b/src/proxy/processors/push-action/index.js index 72a97b33..927fd06f 100644 --- a/src/proxy/processors/push-action/index.js +++ b/src/proxy/processors/push-action/index.js @@ -11,3 +11,4 @@ exports.checkCommitMessages = require('./checkCommitMessages').exec; exports.checkAuthorEmails = require('./checkAuthorEmails').exec; exports.checkUserPushPermission = require('./checkUserPushPermission').exec; exports.clearBareClone = require('./clearBareClone').exec; +exports.checkSensitiveData = require('./checkSensitiveData').exec; \ No newline at end of file diff --git a/test/CheckSensitive.test.js b/test/CheckSensitive.test.js new file mode 100644 index 00000000..2e4c8421 --- /dev/null +++ b/test/CheckSensitive.test.js @@ -0,0 +1,59 @@ +const path = require('path'); +const { exec } = require('../src/proxy/processors/push-action/checkSensitiveData.js'); // Adjust path as necessary +const sinon = require('sinon'); + +describe('Sensitive Data Detection', () => { + let logStub; + + beforeEach(() => { + logStub = sinon.stub(console, 'log'); // Stub console.log before each test + }); + + afterEach(() => { + logStub.restore(); // Restore console.log after each test + }); + + it('should detect sensitive data in CSV file and block execution', async () => { + // Set up the action with the correct file path + const action = { + steps: [{ + stepName: 'diff', + content: { + filePaths: [path.join(__dirname, 'test_data/sensitive_data.csv')] // Ensure this path is correct + } + }] + }; + + // Call exec with necessary parameters + await exec(null, action); // Ensure exec is awaited if it's a promise + + // Capture logged messages for debugging after exec execution + const loggedMessages = logStub.getCalls().map(call => call.args[0]); + console.log('Captured log messages for CSV:', loggedMessages); + + // Check if the blocking message is logged + sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); + }); + + it('should detect sensitive data in XLSX file and block execution', async () => { + // Set up the action with the correct file path for XLSX + const action = { + steps: [{ + stepName: 'diff', + content: { + filePaths: [path.join(__dirname, 'test_data/sensitive_data2.xlsx')] // Ensure this path is correct + } + }] + }; + + // Call exec with necessary parameters + await exec(null, action); // Ensure exec is awaited if it's a promise + + // Capture logged messages for debugging after exec execution + const loggedMessages = logStub.getCalls().map(call => call.args[0]); + console.log('Captured log messages for XLSX:', loggedMessages); + + // Check if the blocking message is logged + sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); + }); +}); diff --git a/test/CreateExcel.js b/test/CreateExcel.js new file mode 100644 index 00000000..be004b13 --- /dev/null +++ b/test/CreateExcel.js @@ -0,0 +1,24 @@ +const XLSX = require('xlsx'); +const fs = require('fs'); +const path = require('path'); + +// Example data with sensitive information +const data = [ + { Name: "John Doe", SSN: "123-45-6789", Email: "john@example.com" }, + { Name: "Jane Smith", SSN: "987-65-4321", Email: "jane@example.com" } +]; + +const worksheet = XLSX.utils.json_to_sheet(data); +const workbook = XLSX.utils.book_new(); +XLSX.utils.book_append_sheet(workbook, worksheet, "SensitiveData"); + +// Create the path to the test_data directory +const testDataPath = path.join(__dirname, 'test_data'); // Ensure this points to the correct directory + +// Create the test_data directory if it doesn't exist +if (!fs.existsSync(testDataPath)){ + fs.mkdirSync(testDataPath, { recursive: true }); // Using recursive to ensure all directories are created +} + +// Write the Excel file to the test_data directory +XLSX.writeFile(workbook, path.join(testDataPath, 'sensitive_data2.xlsx')); diff --git a/test/chain.test.js b/test/chain.test.js index 33d5750a..97515fef 100644 --- a/test/chain.test.js +++ b/test/chain.test.js @@ -1,6 +1,7 @@ const chai = require('chai'); const sinon = require('sinon'); const { PluginLoader } = require('../src/plugin'); +const { checkSensitiveData } = require('../src/proxy/processors/push-action'); chai.should(); const expect = chai.expect; @@ -25,6 +26,7 @@ const mockPushProcessors = { pullRemote: sinon.stub(), writePack: sinon.stub(), getDiff: sinon.stub(), + checkSensitiveData : sinon.stub(), clearBareClone: sinon.stub(), scanDiff: sinon.stub(), blockForAuth: sinon.stub(), @@ -39,6 +41,7 @@ mockPushProcessors.checkIfWaitingAuth.displayName = 'checkIfWaitingAuth'; mockPushProcessors.pullRemote.displayName = 'pullRemote'; mockPushProcessors.writePack.displayName = 'writePack'; mockPushProcessors.getDiff.displayName = 'getDiff'; +mockPushProcessors.checkSensitiveData.displayName = 'checkSensitiveData'; mockPushProcessors.clearBareClone.displayName = 'clearBareClone'; mockPushProcessors.scanDiff.displayName = 'scanDiff'; mockPushProcessors.blockForAuth.displayName = 'blockForAuth'; @@ -106,6 +109,7 @@ describe('proxy chain', function () { mockPushProcessors.checkCommitMessages.resolves(continuingAction); mockPushProcessors.checkAuthorEmails.resolves(continuingAction); mockPushProcessors.checkUserPushPermission.resolves(continuingAction); + mockPushProcessors.checkSensitiveData.resolves(continuingAction); // this stops the chain from further execution mockPushProcessors.checkIfWaitingAuth.resolves({ type: 'push', continue: () => false, allowPush: false }); @@ -120,6 +124,7 @@ describe('proxy chain', function () { expect(mockPushProcessors.checkIfWaitingAuth.called).to.be.true; expect(mockPushProcessors.pullRemote.called).to.be.false; expect(mockPushProcessors.audit.called).to.be.true; + expect(mockPushProcessors.checkSensitiveData.called).to.be.false; expect(result.type).to.equal('push'); expect(result.allowPush).to.be.false; @@ -135,6 +140,7 @@ describe('proxy chain', function () { mockPushProcessors.checkCommitMessages.resolves(continuingAction); mockPushProcessors.checkAuthorEmails.resolves(continuingAction); mockPushProcessors.checkUserPushPermission.resolves(continuingAction); + mockPushProcessors.checkSensitiveData.resolves(continuingAction); // this stops the chain from further execution mockPushProcessors.checkIfWaitingAuth.resolves({ type: 'push', continue: () => true, allowPush: true }); const result = await chain.executeChain(req); @@ -148,6 +154,7 @@ describe('proxy chain', function () { expect(mockPushProcessors.checkIfWaitingAuth.called).to.be.true; expect(mockPushProcessors.pullRemote.called).to.be.false; expect(mockPushProcessors.audit.called).to.be.true; + expect(mockPushProcessors.checkSensitiveData.called).to.be.false; expect(result.type).to.equal('push'); expect(result.allowPush).to.be.true; @@ -170,6 +177,7 @@ describe('proxy chain', function () { mockPushProcessors.clearBareClone.resolves(continuingAction); mockPushProcessors.scanDiff.resolves(continuingAction); mockPushProcessors.blockForAuth.resolves(continuingAction); + mockPushProcessors.checkSensitiveData.resolves(continuingAction); const result = await chain.executeChain(req); @@ -187,6 +195,7 @@ describe('proxy chain', function () { expect(mockPushProcessors.scanDiff.called).to.be.true; expect(mockPushProcessors.blockForAuth.called).to.be.true; expect(mockPushProcessors.audit.called).to.be.true; + expect(mockPushProcessors.checkSensitiveData.called).to.be.true; expect(result.type).to.equal('push'); expect(result.allowPush).to.be.false; diff --git a/test/test_data/sensitive_data.csv b/test/test_data/sensitive_data.csv new file mode 100644 index 00000000..d0381fa0 --- /dev/null +++ b/test/test_data/sensitive_data.csv @@ -0,0 +1,3 @@ +Name,SSN,Email +John Doe,123-45-6789,john@example.com +Jane Smith,987-65-4321,jane@example.com diff --git a/test/test_data/sensitive_data2.xlsx b/test/test_data/sensitive_data2.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..3c9634cc7ee5f00e3e8bf7bbaeb8b19f615f1157 GIT binary patch literal 16196 zcmeHO*>5CA8FxrP(V{3sfP|2cS|br51-m_-vDbEN2YYO1FK;Z5cXJ>VYPxG?+U_ge zJw6g5B^wm2h(IEQ>j8-eq#z++r38_O@E7n7Pw=vvL=aG3ctPTU-&fUF_4JId5hxI^ zX1lxkyXw2@tD`qB-+j+##*)9E-Sy44e)F-hG5UQTjR&^=binOU-}i#umgntG960s_ zwJbb>J8h2#q2;-YmFmP~MdPmFnU>pGtX$h(AD^vgVZ>aM*`CW6D~CL+JhF7(xrHrm zvj|9G*YZORRNQc}(v2d2Ue`mT%N-U@cs_UWrtJj|i|`n9be|c!ti$!G$;lZV5@a+g zOA1ZxqPbWJE}GSfwteV>^U<`ucH1)es%P{Z?nWgnME}Ds=Mjjqpu?lZ3aLaig}S&@ zNjDX}T$rg*3KKy|JQGAb>f-;QGSo&XLl_;}uxP3c`8-So^^vkCTWs)%nJi+N6i(cd z$e}CFwzR1@`{eyMcmMo_u`&96S~`1Cm8bV|HZN#N|DI>Z@j^3T`-mRLmJaAJ%dKcG zbKt^{2?y1-h2fVLV&ZEy{J@PuYB21eNx{Pv41@-oH+QRa2@k`D9cX*ZhVNRg>Fsax z1NgJe!U!)GE0f}nJXlYW>}UXP4sTuj#dtA-xjQ9z2Ax*Qw7OoOJHMLJ7XWtD54X0q z)>x~i^aKDi4D?jWlBZc;pRKMWG;=`mwx4!ma(c3^!sqDM`edG4Sy`E$i)E2S1(3Y0 z_rcFj&eWHu6nFt7;ivoH>nqER#*6|lfF%4(AN>0HxtY2OFMztX8DU=&@zO#guzY0POFlzHWb{6G`diO`) z`O7a}D*;~)SSt^{Z8<#DuJV0t%X82-p%_(i-U?2JvE5~sg2B3=T|#;->MDR&51CyC zx58EBT@R2v$^akj?I^M~yFo9qN;qHcItt*1=h-V>P^y5-#4D%a?Vj5yXCCzOa9fP+ zmGN#cS268c&qvlwoKn$!qstXBH*MxdC_!8vY4pz9<-DYKPg<6u*A2@EywGb$+LM;H z!mLu2Y+EhGkuszUmIJ#Smh~B?R&=v*U0d<&vR<$9JrxoG!9=kl!#NGHZEh=Se3bPf z=9H1#W~g>@hPlF`ZkdqH!@$VHt%VUR-Ql*Ut(iOw%WxhG(BB;G%MAId)a)Dfp`$_u zk+oY!^9u94Jmjjk+vqaKFN1AbZZ{8mG2DfVGHugCvC$u-8=e{=^b9*Ncc8hiTU1GA zdE@9raLwu{B4rLjZ+bzASdVf~@!#g5-DX^58KiV2Q#+PBEN$Bs^0wsgQQBUtEMpKW z3!V#wWIYJ(LztphSCvi|0qL$l%XsC^BH!jnhFwGYm3U;8CZC? zE!%F=ocu~CYVQyOkhzXV!V}9XJe`R7UEES6gMk9q37Ejq0xx>pin`4%qXm?zm_~NO zm~WDdQ!l*_Bj?1ie zJ|>F3orYPJggQ32st`>iMTJ}1Wm2)2mqX=g*U*@16P1mIdI(yc!O|A7Mlr^0jzz3w z4xE^Uw3^(H*hF!!WkEhP$VYHBHL1uTbC~8E3^x7k1Q3Pq$ z<3}A%EGR+p?DRW%65s9lz(cvPR=%(4wNL5<4IAQg^T>xq=1j8 zrseH6H=?|;b1kGh`50$HuNU#4*){jImfZ`s7|aiyQh!V<#DawgXnMfv8fja+QgdsZ zY4U4Y0#3Fw+wZbCThi)G5^)k-xN0hcc*==~zLm-@I(PD}P{fa_D|t8mKkEt&2UtO< znPdS2i@O2SC{h+HUJ!LXWFfz68S4SYeUUrhEm)z&YbYNZDdHONJ-W)6lQcwFVYC4M*m^_vjW5B!i{M_ z-S)PSTQzJyNd0N>>T$|y@E}T%@~(xtM>a}lB2rSdadApoO9tx^S3xP?g0e}nZO(^zkI( zz~2mnceOn8Fut`zIp9jKqd5#g+U^!_YnFMC%s*01$Pkh{BuR-Zpb3o}Ek&%AO&V0h z7@kn&Xqh5U9gUn~2+l~^DY95lDFL5{3){ih{&CN1Pu(*%M!);y zc5o;*xoI=`^c%r~Mw7OykAK!7^aIXJajHPw4Hl;*=J${|Pj{%n^o-5G^TX@g*pv(5 zpKi}tccM@;yq*iHWI)-Z$qSvOn4YO<@+3yXGqD`-;4?ihdic&;-+JfUH}yMjy@HD- z+qzwOefP&VpM4JmN#K2ukXG7YK$YIAr@n4fr3Jjh8mU@!e?dBSH;$<;n`lG z2^+?fWZET{5Djb<#EaE!Y+TrfvRU9s$CM2+K*<{<2XVow1LcEM0E!}NSX#g+gi_yK z#{&&+bWbW=thhMbBDx^qj)2Cvby&r;QScRnb;1DOmKT^fOp`Z+Y+72dd7Io8Rb7{^ zk>}(8mKQ~ygB#Og9nWPpZuJBsz5!W-$JNynZH4whJ7-{wzoZ1RHQhp>_*T-B4@5st zP0~IyS?!=5Gh@DQA71S_EshOjI`Nc=ZsFDk5Y0YOGl_DPM3{}r?^aPhr4v1Obd7$+bq@jMvc@Z z5+SzC_F=E|UQ3Ro(TmKs>H7qB6$gY6Jl;Oln3UfPx>?5QFgO_YBGIz9bQSqyLEj@6 zj{#OxfM)Zm`o5vQU&Cn~JB84}F6Bu{MDmm>|9D+ z5NoJ#J0SKIet`W?lL9NEP(36?!CA0e_b)M*YfT3$zA2snTT#L0X3vk$Opn)V zQ&klXX3CS>0l6W<$m4R-E>;H{kK9}f0xy8Y@{g3DkPDg-0W61fScwd%h0mzoq34l6 zBRi{Ch^?=`F!th2#1;MSm$8*UFLwHnN~nx1q7Li;xwsHiPI8Wb?5oQoi+ZtAsGipY z%jn8eak5%CF#AuR9ha7Qh>_JYSb%Y>vUE<<7LvY-0wOnB;a%ZvoQ|KkvbwyvRnaiu zHhaF0RSs@mPmhhpD`bY}9}468IE==ZZOjp2Y4LEK**%Wk20UZ2q$9Q?w9vP$Xx$R) zCU}8xYj}3EYqh0|VAjdvK}2GThKA>e<8p!)3q$ygxHUW;(D5V?qOoKHX9(#*7yZ?h z4vVICFX{!Hz(~!7Zf1p))JSWWaD;N;z_jWSKSEn*(0YNjSb45xwpz1>X^gj7m5mYP?t_7-y6dRg(^v&Ml;R zE(GllFK+(+;$357^oxTwN&IK4%M&Ui3)+zlgNbCD0piQ#G&Z5y=vv|9@geMVOiaeO z*aw?5dPdeBr`(D}Ns|s587V^ME!>Dx(NxHZS4pmf5j-*Y#+t%5~UQG3{cTU^RiLg=3(>qR(*-8zi1ve zDj+pVLGVHqaodXUwZLQTO%~9CUhz5uY#_C1po{`jP%-8YxM?7uezx8Pm{t z{7Habov7-V1Jl}(GKD?}QM<(b+TY)w*slpO zP}5cQCpNB#H8(bHx%3&3L8~O{_ZL6^Gu(KKPi8Qmc5HFT%Yn0vPCsnXrMT=EJy;-av4}AnY%ad8ngIT){L*Cr137sa+JoU798^vALxN0s7g~;AP!W#s;oCB5FURodREHvx z#Gv+ajE>bV#h~_b4DOU;9E{VOa-98vIP|6*XU8w^%Z|?nbw<1@>;Dj(Xn+FHK1ee0 zs*KGL9N#QzEgvTIwp!Y z(22MeqDGnM$AL>hdJO(@e9wQRI66V7z!UV3kV7oi=f6=riwJ_)5e&1fo0A!NxZ1^j z&@j$v&6*SwC*z*DSYgg_EROJAQD&b-vn2)YFy3(Rq)s_A>kKcXh!7k;0HQICoU2w?jU9a{-p^5nNt=~i|hUw>Wg0#sA-_( zzcwyXDXws4RQ~iOK_zdkB64x1B}0Do%c%&-H{eC&;(|ejeD8#^d{9IzPWm#$=O_E8 tfJN+LEuUfEnaW`&!B|8uR-hTWQ!lMYH!pwu69jY)|DJ Date: Sat, 26 Oct 2024 22:45:57 +0530 Subject: [PATCH 2/9] feat: add support for .log and .json files --- .husky/commit-msg | 3 +- src/proxy/chain.js | 2 +- .../push-action/checkSensitiveData.js | 43 ++++++++------- test/CheckSensitive.test.js | 54 +++++++++++-------- test/CreateExcel.js | 7 +-- test/chain.test.js | 2 +- test/test_data/sensitive_data4.json | 4 ++ 7 files changed, 64 insertions(+), 51 deletions(-) create mode 100644 test/test_data/sensitive_data4.json diff --git a/.husky/commit-msg b/.husky/commit-msg index 53b8922a..22cbe859 100755 --- a/.husky/commit-msg +++ b/.husky/commit-msg @@ -1,4 +1,3 @@ -#!/usr/bin/env sh -. "$(dirname -- "$0")/_/husky.sh" + npx --no -- commitlint --edit ${1} && npm run lint diff --git a/src/proxy/chain.js b/src/proxy/chain.js index 332e5379..05b8c7e2 100644 --- a/src/proxy/chain.js +++ b/src/proxy/chain.js @@ -10,7 +10,7 @@ const pushActionChain = [ proc.push.pullRemote, proc.push.writePack, proc.push.getDiff, - proc.push.checkSensitiveData, //checkSensitiveData added + proc.push.checkSensitiveData, // checkSensitiveData added proc.push.clearBareClone, proc.push.scanDiff, proc.push.blockForAuth, diff --git a/src/proxy/processors/push-action/checkSensitiveData.js b/src/proxy/processors/push-action/checkSensitiveData.js index a33614e1..f082a8a3 100644 --- a/src/proxy/processors/push-action/checkSensitiveData.js +++ b/src/proxy/processors/push-action/checkSensitiveData.js @@ -2,8 +2,7 @@ const fs = require('fs'); const csv = require('csv-parser'); const XLSX = require('xlsx'); const path = require('path'); -const { exec: getDiffExec } = require('./getDiff'); - +// const { exec: getDiffExec } = require('./getDiff'); // Function to check for sensitive data patterns const checkForSensitiveData = (cell) => { const sensitivePatterns = [ @@ -20,12 +19,10 @@ const checkForSensitiveData = (cell) => { return false; }); }; - // Function to process CSV files const processCSV = async (filePath) => { return new Promise((resolve, reject) => { let sensitiveDataFound = false; - fs.createReadStream(filePath) .pipe(csv()) .on('data', (row) => { @@ -48,18 +45,15 @@ const processCSV = async (filePath) => { }); }); }; - // Function to process XLSX files const processXLSX = async (filePath) => { return new Promise((resolve, reject) => { let sensitiveDataFound = false; - try { const workbook = XLSX.readFile(filePath); const sheetName = workbook.SheetNames[0]; const sheet = workbook.Sheets[sheetName]; const jsonData = XLSX.utils.sheet_to_json(sheet); - jsonData.forEach((row) => { for (const [key, value] of Object.entries(row)) { if (checkForSensitiveData(value)) { @@ -68,7 +62,6 @@ const processXLSX = async (filePath) => { } } }); - if (!sensitiveDataFound) { console.log('No sensitive data found in XLSX.'); } @@ -79,7 +72,23 @@ const processXLSX = async (filePath) => { } }); }; - +// Function to check for sensitive data in .log and .json files +const checkLogJsonFiles = async (filePath) => { + return new Promise((resolve, reject) => { + let sensitiveDataFound = false; + fs.readFile(filePath, 'utf8', (err, data) => { + if (err) { + console.error(`Error reading file ${filePath}: ${err.message}`); + return reject(err); + } + if (checkForSensitiveData(data)) { + console.log(`\x1b[33mSensitive data found in ${filePath}\x1b[0m`); + sensitiveDataFound = true; + } + resolve(sensitiveDataFound); + }); + }); +}; // Function to parse the file based on its extension const parseFile = async (filePath) => { const ext = path.extname(filePath).toLowerCase(); @@ -89,28 +98,26 @@ const parseFile = async (filePath) => { return await processCSV(filePath); case '.xlsx': return await processXLSX(filePath); + case '.log': + return await checkLogJsonFiles(filePath); + case '.json': + return await checkLogJsonFiles(filePath); default: - console.log(`Unsupported file type: ${ext} for file: ${filePath}`); + // Skip unsupported file types without logging return false; // Indicate that no sensitive data was found for unsupported types } }; - // Async exec function to handle actions const exec = async (req, action) => { // getDiffExec(req, action); // Call to getDiffExec if necessary - const diffStep = action.steps.find((s) => s.stepName === 'diff'); - if (diffStep && diffStep.content) { console.log('Diff content:', diffStep.content); - const filePaths = diffStep.content.filePaths || []; - if (filePaths.length > 0) { // Check for sensitive data in all files const sensitiveDataFound = await Promise.all(filePaths.map(parseFile)); const anySensitiveDataDetected = sensitiveDataFound.some(found => found); // Check if any file reported sensitive data - if (anySensitiveDataDetected) { action.pushBlocked = true; // Block the push action.error = true; // Set error flag @@ -123,9 +130,7 @@ const exec = async (req, action) => { } else { console.log('No diff content available.'); } - return action; // Returning action for testing purposes }; - exec.displayName = 'logFileChanges.exec'; -exports.exec = exec; +exports.exec = exec; \ No newline at end of file diff --git a/test/CheckSensitive.test.js b/test/CheckSensitive.test.js index 2e4c8421..87340d9d 100644 --- a/test/CheckSensitive.test.js +++ b/test/CheckSensitive.test.js @@ -1,20 +1,15 @@ const path = require('path'); const { exec } = require('../src/proxy/processors/push-action/checkSensitiveData.js'); // Adjust path as necessary const sinon = require('sinon'); - describe('Sensitive Data Detection', () => { let logStub; - beforeEach(() => { logStub = sinon.stub(console, 'log'); // Stub console.log before each test }); - afterEach(() => { logStub.restore(); // Restore console.log after each test }); - it('should detect sensitive data in CSV file and block execution', async () => { - // Set up the action with the correct file path const action = { steps: [{ stepName: 'diff', @@ -23,20 +18,12 @@ describe('Sensitive Data Detection', () => { } }] }; - - // Call exec with necessary parameters - await exec(null, action); // Ensure exec is awaited if it's a promise - - // Capture logged messages for debugging after exec execution + await exec(null, action); const loggedMessages = logStub.getCalls().map(call => call.args[0]); console.log('Captured log messages for CSV:', loggedMessages); - - // Check if the blocking message is logged sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); - it('should detect sensitive data in XLSX file and block execution', async () => { - // Set up the action with the correct file path for XLSX const action = { steps: [{ stepName: 'diff', @@ -45,15 +32,38 @@ describe('Sensitive Data Detection', () => { } }] }; - - // Call exec with necessary parameters - await exec(null, action); // Ensure exec is awaited if it's a promise - - // Capture logged messages for debugging after exec execution + await exec(null, action); const loggedMessages = logStub.getCalls().map(call => call.args[0]); console.log('Captured log messages for XLSX:', loggedMessages); - - // Check if the blocking message is logged sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); -}); + it('should detect sensitive data in a log file and block execution', async () => { + const action = { + steps: [{ + stepName: 'diff', + content: { + filePaths: [path.join(__dirname, 'test_data/sensitive_data3.log')] // Ensure this path is correct + } + }] + }; + await exec(null, action); + const loggedMessages = logStub.getCalls().map(call => call.args[0]); + console.log('Captured log messages for log file:', loggedMessages); + sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); + }); + it('should detect sensitive data in a JSON file and block execution', async () => { + const action = { + steps: [{ + stepName: 'diff', + content: { + filePaths: [path.join(__dirname, 'test_data/sensitive_data4.json')] // Ensure this path is correct + } + }] + }; + await exec(null, action); + const loggedMessages = logStub.getCalls().map(call => call.args[0]); + console.log('Captured log messages for JSON file:', loggedMessages); + sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); + }); + +}); \ No newline at end of file diff --git a/test/CreateExcel.js b/test/CreateExcel.js index be004b13..ea9f9af0 100644 --- a/test/CreateExcel.js +++ b/test/CreateExcel.js @@ -1,24 +1,19 @@ const XLSX = require('xlsx'); const fs = require('fs'); const path = require('path'); - // Example data with sensitive information const data = [ { Name: "John Doe", SSN: "123-45-6789", Email: "john@example.com" }, { Name: "Jane Smith", SSN: "987-65-4321", Email: "jane@example.com" } ]; - const worksheet = XLSX.utils.json_to_sheet(data); const workbook = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(workbook, worksheet, "SensitiveData"); - // Create the path to the test_data directory const testDataPath = path.join(__dirname, 'test_data'); // Ensure this points to the correct directory - // Create the test_data directory if it doesn't exist if (!fs.existsSync(testDataPath)){ fs.mkdirSync(testDataPath, { recursive: true }); // Using recursive to ensure all directories are created } - // Write the Excel file to the test_data directory -XLSX.writeFile(workbook, path.join(testDataPath, 'sensitive_data2.xlsx')); +XLSX.writeFile(workbook, path.join(testDataPath, 'sensitive_data2.xlsx')); \ No newline at end of file diff --git a/test/chain.test.js b/test/chain.test.js index 97515fef..beeea18a 100644 --- a/test/chain.test.js +++ b/test/chain.test.js @@ -1,7 +1,7 @@ const chai = require('chai'); const sinon = require('sinon'); const { PluginLoader } = require('../src/plugin'); -const { checkSensitiveData } = require('../src/proxy/processors/push-action'); + chai.should(); const expect = chai.expect; diff --git a/test/test_data/sensitive_data4.json b/test/test_data/sensitive_data4.json new file mode 100644 index 00000000..50217f69 --- /dev/null +++ b/test/test_data/sensitive_data4.json @@ -0,0 +1,4 @@ +{ + "username": "johndoe", + "ssn": "123-45-6789" +} From 8b257a22ce03bff7ad63ba2018190bc8e7f2b8a3 Mon Sep 17 00:00:00 2001 From: ChaitanyaD48 Date: Sun, 27 Oct 2024 08:22:31 +0530 Subject: [PATCH 3/9] test: add test for edge cases Signed-off-by: ChaitanyaD48 --- test/CheckSensitive.test.js | 60 ++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/test/CheckSensitive.test.js b/test/CheckSensitive.test.js index 87340d9d..e1ca2b16 100644 --- a/test/CheckSensitive.test.js +++ b/test/CheckSensitive.test.js @@ -1,14 +1,18 @@ const path = require('path'); const { exec } = require('../src/proxy/processors/push-action/checkSensitiveData.js'); // Adjust path as necessary const sinon = require('sinon'); + describe('Sensitive Data Detection', () => { let logStub; + beforeEach(() => { logStub = sinon.stub(console, 'log'); // Stub console.log before each test }); + afterEach(() => { logStub.restore(); // Restore console.log after each test }); + it('should detect sensitive data in CSV file and block execution', async () => { const action = { steps: [{ @@ -19,10 +23,9 @@ describe('Sensitive Data Detection', () => { }] }; await exec(null, action); - const loggedMessages = logStub.getCalls().map(call => call.args[0]); - console.log('Captured log messages for CSV:', loggedMessages); sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); + it('should detect sensitive data in XLSX file and block execution', async () => { const action = { steps: [{ @@ -33,10 +36,9 @@ describe('Sensitive Data Detection', () => { }] }; await exec(null, action); - const loggedMessages = logStub.getCalls().map(call => call.args[0]); - console.log('Captured log messages for XLSX:', loggedMessages); sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); + it('should detect sensitive data in a log file and block execution', async () => { const action = { steps: [{ @@ -47,10 +49,9 @@ describe('Sensitive Data Detection', () => { }] }; await exec(null, action); - const loggedMessages = logStub.getCalls().map(call => call.args[0]); - console.log('Captured log messages for log file:', loggedMessages); sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); + it('should detect sensitive data in a JSON file and block execution', async () => { const action = { steps: [{ @@ -61,9 +62,48 @@ describe('Sensitive Data Detection', () => { }] }; await exec(null, action); - const loggedMessages = logStub.getCalls().map(call => call.args[0]); - console.log('Captured log messages for JSON file:', loggedMessages); sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); - -}); \ No newline at end of file + + it('should allow execution if no sensitive data is found', async () => { + const action = { + steps: [{ + stepName: 'diff', + content: { + filePaths: [path.join(__dirname, 'test_data/no_sensitive_data.txt')] // Ensure this path is correct + } + }] + }; + await exec(null, action); + sinon.assert.neverCalledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); + }); + + it('should allow execution for an empty file', async () => { + const action = { + steps: [{ + stepName: 'diff', + content: { + filePaths: [path.join(__dirname, 'test_data/empty_file.txt')] // Ensure this path is correct + } + }] + }; + await exec(null, action); + sinon.assert.neverCalledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); + }); + + it('should handle file-not-found scenario gracefully', async () => { + const action = { + steps: [{ + stepName: 'diff', + content: { + filePaths: [path.join(__dirname, 'test_data/non_existent_file.txt')] // Ensure this path is correct + } + }] + }; + try { + await exec(null, action); + } catch (error) { + sinon.assert.match(error.message, /ENOENT: no such file or directory/); + } + }); +}); From 027a45903b43aa43a88b8b2eb78d8066db00b2cb Mon Sep 17 00:00:00 2001 From: Prachit Date: Mon, 28 Oct 2024 10:53:01 +0530 Subject: [PATCH 4/9] refactor: corrected the way filepaths are geting extracted from diff --- .../push-action/checkSensitiveData.js | 35 +++++++++++++++---- test/CheckSensitive.test.js | 35 +++++++------------ 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/proxy/processors/push-action/checkSensitiveData.js b/src/proxy/processors/push-action/checkSensitiveData.js index f082a8a3..f6d9326d 100644 --- a/src/proxy/processors/push-action/checkSensitiveData.js +++ b/src/proxy/processors/push-action/checkSensitiveData.js @@ -108,20 +108,39 @@ const parseFile = async (filePath) => { } }; // Async exec function to handle actions +// Function to parse file paths from git diff content +const extractFilePathsFromDiff = (diffContent) => { + const filePaths = []; + const lines = diffContent.split('\n'); + + lines.forEach(line => { + const match = line.match(/^diff --git a\/(.+?) b\/(.+?)$/); + if (match) { + filePaths.push(match[1]); // Extract the file path from "a/" in the diff line + } + }); + + return filePaths; +}; + const exec = async (req, action) => { - // getDiffExec(req, action); // Call to getDiffExec if necessary const diffStep = action.steps.find((s) => s.stepName === 'diff'); + if (diffStep && diffStep.content) { console.log('Diff content:', diffStep.content); - const filePaths = diffStep.content.filePaths || []; + + // Use the parsing function to get file paths + const filePaths = extractFilePathsFromDiff(diffStep.content); + if (filePaths.length > 0) { // Check for sensitive data in all files const sensitiveDataFound = await Promise.all(filePaths.map(parseFile)); - const anySensitiveDataDetected = sensitiveDataFound.some(found => found); // Check if any file reported sensitive data + const anySensitiveDataDetected = sensitiveDataFound.some(found => found); + if (anySensitiveDataDetected) { - action.pushBlocked = true; // Block the push - action.error = true; // Set error flag - action.errorMessage = 'Your push has been blocked due to sensitive data detection.'; // Set error message + action.pushBlocked = true; + action.error = true; + action.errorMessage = 'Your push has been blocked due to sensitive data detection.'; console.log(action.errorMessage); } } else { @@ -130,7 +149,11 @@ const exec = async (req, action) => { } else { console.log('No diff content available.'); } + return action; // Returning action for testing purposes }; + + + exec.displayName = 'logFileChanges.exec'; exports.exec = exec; \ No newline at end of file diff --git a/test/CheckSensitive.test.js b/test/CheckSensitive.test.js index e1ca2b16..f26324cd 100644 --- a/test/CheckSensitive.test.js +++ b/test/CheckSensitive.test.js @@ -1,4 +1,4 @@ -const path = require('path'); +// const path = require('path'); const { exec } = require('../src/proxy/processors/push-action/checkSensitiveData.js'); // Adjust path as necessary const sinon = require('sinon'); @@ -13,13 +13,16 @@ describe('Sensitive Data Detection', () => { logStub.restore(); // Restore console.log after each test }); + const createDiffContent = (filePaths) => { + // Format file paths in diff format + return filePaths.map(filePath => `diff --git a/${filePath} b/${filePath}`).join('\n'); + }; + it('should detect sensitive data in CSV file and block execution', async () => { const action = { steps: [{ stepName: 'diff', - content: { - filePaths: [path.join(__dirname, 'test_data/sensitive_data.csv')] // Ensure this path is correct - } + content: createDiffContent(['test/test_data/sensitive_data.csv']) // Ensure this path is correct }] }; await exec(null, action); @@ -30,9 +33,7 @@ describe('Sensitive Data Detection', () => { const action = { steps: [{ stepName: 'diff', - content: { - filePaths: [path.join(__dirname, 'test_data/sensitive_data2.xlsx')] // Ensure this path is correct - } + content: createDiffContent(['test/test_data/sensitive_data2.xlsx']) // Ensure this path is correct }] }; await exec(null, action); @@ -43,9 +44,7 @@ describe('Sensitive Data Detection', () => { const action = { steps: [{ stepName: 'diff', - content: { - filePaths: [path.join(__dirname, 'test_data/sensitive_data3.log')] // Ensure this path is correct - } + content: createDiffContent(['test/test_data/sensitive_data3.log']) // Ensure this path is correct }] }; await exec(null, action); @@ -56,9 +55,7 @@ describe('Sensitive Data Detection', () => { const action = { steps: [{ stepName: 'diff', - content: { - filePaths: [path.join(__dirname, 'test_data/sensitive_data4.json')] // Ensure this path is correct - } + content: createDiffContent(['test/test_data/sensitive_data4.json']) // Ensure this path is correct }] }; await exec(null, action); @@ -69,9 +66,7 @@ describe('Sensitive Data Detection', () => { const action = { steps: [{ stepName: 'diff', - content: { - filePaths: [path.join(__dirname, 'test_data/no_sensitive_data.txt')] // Ensure this path is correct - } + content: createDiffContent(['test_data/no_sensitive_data.txt']) // Ensure this path is correct }] }; await exec(null, action); @@ -82,9 +77,7 @@ describe('Sensitive Data Detection', () => { const action = { steps: [{ stepName: 'diff', - content: { - filePaths: [path.join(__dirname, 'test_data/empty_file.txt')] // Ensure this path is correct - } + content: createDiffContent(['test_data/empty_file.txt']) // Ensure this path is correct }] }; await exec(null, action); @@ -95,9 +88,7 @@ describe('Sensitive Data Detection', () => { const action = { steps: [{ stepName: 'diff', - content: { - filePaths: [path.join(__dirname, 'test_data/non_existent_file.txt')] // Ensure this path is correct - } + content: createDiffContent(['test_data/non_existent_file.txt']) // Ensure this path is correct }] }; try { From df8254804cd7cee219d7e6e7c9dd56a0f34d6f3c Mon Sep 17 00:00:00 2001 From: Prachit Date: Mon, 28 Oct 2024 12:36:59 +0530 Subject: [PATCH 5/9] refactor: modified the test to provide exact action and diff content --- .../push-action/checkSensitiveData.js | 12 +-- test/CheckSensitive.test.js | 84 +++++++++---------- 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/proxy/processors/push-action/checkSensitiveData.js b/src/proxy/processors/push-action/checkSensitiveData.js index f6d9326d..69faef37 100644 --- a/src/proxy/processors/push-action/checkSensitiveData.js +++ b/src/proxy/processors/push-action/checkSensitiveData.js @@ -2,6 +2,7 @@ const fs = require('fs'); const csv = require('csv-parser'); const XLSX = require('xlsx'); const path = require('path'); +const Step = require('../../actions').Step; // const { exec: getDiffExec } = require('./getDiff'); // Function to check for sensitive data patterns const checkForSensitiveData = (cell) => { @@ -125,6 +126,7 @@ const extractFilePathsFromDiff = (diffContent) => { const exec = async (req, action) => { const diffStep = action.steps.find((s) => s.stepName === 'diff'); + const step = new Step('checksensitiveData'); if (diffStep && diffStep.content) { console.log('Diff content:', diffStep.content); @@ -138,10 +140,10 @@ const exec = async (req, action) => { const anySensitiveDataDetected = sensitiveDataFound.some(found => found); if (anySensitiveDataDetected) { - action.pushBlocked = true; - action.error = true; - action.errorMessage = 'Your push has been blocked due to sensitive data detection.'; - console.log(action.errorMessage); + step.blocked= true; + step.error = true; + step.errorMessage = 'Your push has been blocked due to sensitive data detection.'; + console.log(step.errorMessage); } } else { console.log('No file paths provided in the diff step.'); @@ -149,7 +151,7 @@ const exec = async (req, action) => { } else { console.log('No diff content available.'); } - + action.addStep(step); return action; // Returning action for testing purposes }; diff --git a/test/CheckSensitive.test.js b/test/CheckSensitive.test.js index f26324cd..3916780d 100644 --- a/test/CheckSensitive.test.js +++ b/test/CheckSensitive.test.js @@ -1,6 +1,8 @@ // const path = require('path'); const { exec } = require('../src/proxy/processors/push-action/checkSensitiveData.js'); // Adjust path as necessary const sinon = require('sinon'); +const {Action}=require('../src/proxy/actions/Action.js') +const {Step}=require('../src/proxy/actions/Step.js') describe('Sensitive Data Detection', () => { let logStub; @@ -19,78 +21,76 @@ describe('Sensitive Data Detection', () => { }; it('should detect sensitive data in CSV file and block execution', async () => { - const action = { - steps: [{ - stepName: 'diff', - content: createDiffContent(['test/test_data/sensitive_data.csv']) // Ensure this path is correct - }] - }; + const action = new Action('action_id', 'push', 'create', Date.now(), 'owner/repo'); + const step = new Step('diff'); + + // Create diff content simulating sensitive data in CSV + step.setContent(createDiffContent(['test/test_data/sensitive_data.csv'])); + action.addStep(step) + await exec(null, action); sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); it('should detect sensitive data in XLSX file and block execution', async () => { - const action = { - steps: [{ - stepName: 'diff', - content: createDiffContent(['test/test_data/sensitive_data2.xlsx']) // Ensure this path is correct - }] - }; + const action = new Action('action_id', 'push', 'create', Date.now(), 'owner/repo'); + const step = new Step('diff'); + step.setContent(createDiffContent(['test/test_data/sensitive_data2.xlsx'])); + action.addStep(step); + await exec(null, action); sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); it('should detect sensitive data in a log file and block execution', async () => { - const action = { - steps: [{ - stepName: 'diff', - content: createDiffContent(['test/test_data/sensitive_data3.log']) // Ensure this path is correct - }] - }; + + const action = new Action('action_id', 'push', 'create', Date.now(), 'owner/repo'); + const step = new Step('diff'); + step.setContent(createDiffContent(['test/test_data/sensitive_data3.log'])); + action.addStep(step); await exec(null, action); sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); it('should detect sensitive data in a JSON file and block execution', async () => { - const action = { - steps: [{ - stepName: 'diff', - content: createDiffContent(['test/test_data/sensitive_data4.json']) // Ensure this path is correct - }] - }; + + + const action = new Action('action_id', 'push', 'create', Date.now(), 'owner/repo'); + const step = new Step('diff'); + step.setContent(createDiffContent(['test/test_data/sensitive_data4.json'])); + action.addStep(step); await exec(null, action); sinon.assert.calledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); it('should allow execution if no sensitive data is found', async () => { - const action = { - steps: [{ - stepName: 'diff', - content: createDiffContent(['test_data/no_sensitive_data.txt']) // Ensure this path is correct - }] - }; + + + const action = new Action('action_id', 'push', 'create', Date.now(), 'owner/repo'); + const step = new Step('diff'); + step.setContent(createDiffContent(['test_data/no_sensitive_data.txt'])); + action.addStep(step); await exec(null, action); sinon.assert.neverCalledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); it('should allow execution for an empty file', async () => { - const action = { - steps: [{ - stepName: 'diff', - content: createDiffContent(['test_data/empty_file.txt']) // Ensure this path is correct - }] - }; + + + const action = new Action('action_id', 'push', 'create', Date.now(), 'owner/repo'); + const step = new Step('diff'); + step.setContent(createDiffContent(['test_data/empty_file.txt'])); + action.addStep(step); await exec(null, action); sinon.assert.neverCalledWith(logStub, sinon.match(/Your push has been blocked due to sensitive data detection/)); }); it('should handle file-not-found scenario gracefully', async () => { - const action = { - steps: [{ - stepName: 'diff', - content: createDiffContent(['test_data/non_existent_file.txt']) // Ensure this path is correct - }] - }; + + const action = new Action('action_id', 'push', 'create', Date.now(), 'owner/repo'); + const step = new Step('diff'); + step.setContent(createDiffContent(['test_data/non_existent_file.txt'])); + action.addStep(step); try { await exec(null, action); } catch (error) { From fd265237fe9f50946509b76e5f2c749f0effaa6f Mon Sep 17 00:00:00 2001 From: Prachit Date: Sun, 10 Nov 2024 21:53:59 +0530 Subject: [PATCH 6/9] refactor: modified proxy.config to support the feature --- proxy.config.json | 3 ++- .../processors/push-action/checkSensitiveData.js | 12 +++++++++++- test/CheckSensitive.test.js | 3 ++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/proxy.config.json b/proxy.config.json index df082a2e..212a7fe9 100644 --- a/proxy.config.json +++ b/proxy.config.json @@ -77,7 +77,8 @@ "block": { "literals": [], "patterns": [], - "providers": {} + "providers": {}, + "proxyFileTypes":[] } } }, diff --git a/src/proxy/processors/push-action/checkSensitiveData.js b/src/proxy/processors/push-action/checkSensitiveData.js index 69faef37..0ea18679 100644 --- a/src/proxy/processors/push-action/checkSensitiveData.js +++ b/src/proxy/processors/push-action/checkSensitiveData.js @@ -3,8 +3,11 @@ const csv = require('csv-parser'); const XLSX = require('xlsx'); const path = require('path'); const Step = require('../../actions').Step; +const config = require('../../../config'); + // const { exec: getDiffExec } = require('./getDiff'); // Function to check for sensitive data patterns +const commitConfig = config.getCommitConfig(); const checkForSensitiveData = (cell) => { const sensitivePatterns = [ /\d{3}-\d{2}-\d{4}/, // Social Security Number (SSN) @@ -83,7 +86,7 @@ const checkLogJsonFiles = async (filePath) => { return reject(err); } if (checkForSensitiveData(data)) { - console.log(`\x1b[33mSensitive data found in ${filePath}\x1b[0m`); + console.log(`\x1b[Sensitive data found in ${filePath}\x1b[0m`); sensitiveDataFound = true; } resolve(sensitiveDataFound); @@ -92,7 +95,14 @@ const checkLogJsonFiles = async (filePath) => { }; // Function to parse the file based on its extension const parseFile = async (filePath) => { + const ext = path.extname(filePath).toLowerCase(); + const FilestoCheck = commitConfig.diff.block.proxyFileTypes; + if(!FilestoCheck.includes(ext)){ + + console.log(`${ext} should be included in CommitConfig for proxy Check!`); + return false; + } switch (ext) { case '.csv': diff --git a/test/CheckSensitive.test.js b/test/CheckSensitive.test.js index 3916780d..f51f79da 100644 --- a/test/CheckSensitive.test.js +++ b/test/CheckSensitive.test.js @@ -4,6 +4,7 @@ const sinon = require('sinon'); const {Action}=require('../src/proxy/actions/Action.js') const {Step}=require('../src/proxy/actions/Step.js') + describe('Sensitive Data Detection', () => { let logStub; @@ -19,7 +20,7 @@ describe('Sensitive Data Detection', () => { // Format file paths in diff format return filePaths.map(filePath => `diff --git a/${filePath} b/${filePath}`).join('\n'); }; - +// make sure the file types are added in proxyfiletypes in proxy.config.json it('should detect sensitive data in CSV file and block execution', async () => { const action = new Action('action_id', 'push', 'create', Date.now(), 'owner/repo'); const step = new Step('diff'); From d9de7664ae707f3220c21f92b1e02b675e489c09 Mon Sep 17 00:00:00 2001 From: Prachit <112982404+Psingle20@users.noreply.github.com> Date: Sun, 1 Dec 2024 12:10:36 +0530 Subject: [PATCH 7/9] Update proxy.config.json Co-authored-by: Thomas Cooper --- proxy.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy.config.json b/proxy.config.json index 9ae3c867..35cd52a9 100644 --- a/proxy.config.json +++ b/proxy.config.json @@ -78,7 +78,7 @@ "literals": [], "patterns": [], "providers": {}, - "proxyFileTypes":[] + "sensitiveFileTypes":[] } } }, From f6798c551ec8ce8b2001376ce8aa3a980de8757b Mon Sep 17 00:00:00 2001 From: Prachit <112982404+Psingle20@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:09:44 +0530 Subject: [PATCH 8/9] Update .gitignore Co-authored-by: Thomas Cooper --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index b398dc87..1849589c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ yarn-debug.log* yarn-error.log* lerna-debug.log* package-lock.json -/git-proxy-test # Diagnostic reports (https://nodejs.org/api/report.html) From 868c074f255d2930abb5effcff1528b65c938dcd Mon Sep 17 00:00:00 2001 From: Prachit <112982404+Psingle20@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:09:54 +0530 Subject: [PATCH 9/9] Update .husky/commit-msg Co-authored-by: Thomas Cooper --- .husky/commit-msg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.husky/commit-msg b/.husky/commit-msg index 22cbe859..53b8922a 100755 --- a/.husky/commit-msg +++ b/.husky/commit-msg @@ -1,3 +1,4 @@ - +#!/usr/bin/env sh +. "$(dirname -- "$0")/_/husky.sh" npx --no -- commitlint --edit ${1} && npm run lint