small changes and restart at midnight (#2312)

integration/monitor/all-scenarios.js

@@ -1,4 +1,5 @@
 const cron = require("node-cron");
+const { exec } = require("child_process");
 const fs = require("fs");
 const path = require("path");
 const Sentry = require("@sentry/node");
@@ -39,12 +40,23 @@ const workers = fs.readdirSync(filesPath).map((file) => {
     workerData: { name, path: path.join(filesPath, file) },
   });
 
+  worker.on("message", ({ exception, metric, screenshot }) => {
+    Sentry.getCurrentScope().addAttachment({
+      filename: `${metric}-${Date.now()}.jpeg`,
+      data: screenshot,
+    });
+
+    Sentry.captureException(exception);
+
+    Sentry.getCurrentScope().clearAttachments();
+  });
+
   return worker;
 });
 
 /*
- * A task runs every minute on the minute between the hours of 5am and midnight
- * EST. Spread out the worker runs over the minute to avoid overwhelming the
+ * A task runs every minute between the hours of 5am and midnight EST.
+ * Spread out the worker runs over the minute to avoid overwhelming the
  * system. If we receive a message from a worker, it means that there was a
  * failure. Capture the exception with Sentry and attach a screenshot to the
  * event.
@@ -55,21 +67,17 @@ cron.schedule("* 5-23 * * *", (_) => {
       (_) => {
         worker.postMessage(null);
       },
-      (60000 / workers.length) * index,
+      (60 * 1000 / workers.length) * index,
     );
-
-    worker.on("message", ({ exception, metric, screenshot }) => {
-      Sentry.getCurrentScope().addAttachment({
-        filename: `${metric}-${Date.now()}.jpeg`,
-        data: screenshot,
-      });
-
-      Sentry.captureException(exception);
-
-      Sentry.getCurrentScope().clearAttachments();
-    });
   });
 }, {
   scheduled: true,
   timezone: "America/New_York"
 });
+
+/*
+ * Restart all the scenarios every day at midnight.
+ */
+cron.schedule("0 0 * * *", (_) => {
+  exec("pm2 restart all-scenarios", (_error, _stdout, _stderr) => {});
+});

integration/monitor/ecosystem.config.js

@@ -4,14 +4,14 @@ module.exports = {
       name: "all-health-checks",
       script: "./integration/monitor/all-health-checks.js",
       instances: 1,
-      max_memory_restart: "256M",
+      max_memory_restart: "128M",
       exec_mode : "cluster"
     },
     {
       name: "all-scenarios",
       script: "./integration/monitor/all-scenarios.js",
       instances: 1,
-      max_memory_restart: "1024M",
+      max_memory_restart: "512M",
       exec_mode : "cluster"
     },
   ],

integration/monitor/worker.js

@@ -13,15 +13,14 @@ const baseURL = process.env.HOST
   ? `https://${process.env.HOST}`
   : "http://localhost:4001";
 
-parentPort.on("message", async (_) => {
-  const { scenario } = require(workerData.path);
+const { scenario } = require(workerData.path);
+const metric = `${prefix}${workerData.name}`;
 
+parentPort.on("message", async (_) => {
   const browser = await chromium.launch();
   const context = await browser.newContext({ userAgent: 'Playwright' });
   const page = await context.newPage();
 
-  const metric = `${prefix}${workerData.name}`;
-
   const start = performance.now();
 
   try {

playwright.config.js

@@ -20,7 +20,7 @@ module.exports = defineConfig({
     trace: 'on-first-retry',
   },
   /* set the expect timeout to 30s */
-  expect: { timeout: 30000 },
+  expect: { timeout: 5000 },
   /* Configure projects for major browsers */
   projects: [
     {