feat(server): add cron job for copilot test

.github/actions/deploy/deploy.mjs

@@ -19,6 +19,8 @@ const {
   COPILOT_FAL_API_KEY,
   COPILOT_PERPLEXITY_API_KEY,
   COPILOT_UNSPLASH_API_KEY,
+  SLACK_BOT_TOKEN,
+  RELEASE_SLACK_CHANNEL_ID,
   MAILER_SENDER,
   MAILER_USER,
   MAILER_PASSWORD,
@@ -150,6 +152,8 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set-string graphql.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`,
     `--set-string graphql.app.copilot.perplexity.key="${COPILOT_PERPLEXITY_API_KEY}"`,
     `--set-string graphql.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`,
+    `--set-string graphql.app.copilot.slack.botToken="${SLACK_BOT_TOKEN}"`,
+    `--set-string graphql.app.copilot.slack.channelId="${RELEASE_SLACK_CHANNEL_ID}"`,
     `--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`,
     `--set-string graphql.app.mailer.user="${MAILER_USER}"`,
     `--set-string graphql.app.mailer.password="${MAILER_PASSWORD}"`,

.github/helm/affine/charts/graphql/templates/copilot-secret.yaml

@@ -9,4 +9,6 @@ data:
   falSecret: {{ .Values.app.copilot.fal.key | b64enc }}
   perplexitySecret: {{ .Values.app.copilot.perplexity.key | b64enc }}
   unsplashSecret: {{ .Values.app.copilot.unsplash.key | b64enc }}
+  slackBotToken: {{ .Values.app.copilot.slack.botToken | b64enc }}
+  slackChannelId: {{ .Values.app.copilot.slack.channelId | b64enc }}
 {{- end }}

.github/helm/affine/charts/graphql/templates/copilot-test.yaml

@@ -0,0 +1,66 @@
+{{ if .Values.app.copilot.enabled }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "graphql.fullname" . }}-copilot-test
+  labels:
+    {{- include "graphql.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  schedule: "0 8 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: {{ include "graphql.serviceAccountName" . }}
+          containers:
+          - name: {{ .Chart.Name }}
+            image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+            command: ["yarn", "test:copilot:e2e:cron"]
+            env:
+              - name: AFFINE_ENV
+                value: "{{ .Release.Namespace }}"
+              - name: SLACK_BOT_TOKEN
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: slackBotToken
+              - name: CHANNEL_ID
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: slackChannelId
+              - name: COPILOT_E2E_ENDPOINT
+                value: "http://{{ include "graphql.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:3000"
+              - name: DATABASE_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: pg-postgresql
+                    key: postgres-password
+              - name: DATABASE_URL
+                value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
+              - name: COPILOT_OPENAI_API_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: openaiSecret
+              - name: COPILOT_FAL_API_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: falSecret
+              - name: COPILOT_UNSPLASH_API_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: "{{ .Values.app.copilot.secretName }}"
+                    key: unsplashSecret
+            resources:
+              requests:
+                cpu: '100m'
+                memory: '200Mi'
+          restartPolicy: Never
+      backoffLimit: 1
+{{ end }}

.github/workflows/copilot-test.yml

@@ -181,7 +181,7 @@ jobs:
         if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
         run: node ./tools/copilot-result/index.js
         env:
-          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           BRANCH_SHA: ${{ github.sha }}
           BRANCH_NAME: ${{ github.ref }}
@@ -191,7 +191,7 @@ jobs:
         if: ${{ always() && contains(needs.*.result, 'failure') }}
         run: node ./tools/copilot-result/index.js
         env:
-          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           BRANCH_SHA: ${{ github.sha }}
           BRANCH_NAME: ${{ github.ref }}
@@ -201,7 +201,7 @@ jobs:
         if: ${{ always() && contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') }}
         run: node ./tools/copilot-result/index.js
         env:
-          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           BRANCH_SHA: ${{ github.sha }}
           BRANCH_NAME: ${{ github.ref }}

.github/workflows/deploy.yml

@@ -100,6 +100,9 @@ jobs:
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
           COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
           COPILOT_UNSPLASH_API_KEY: ${{ secrets.COPILOT_UNSPLASH_API_KEY }}
+          # used for slack notifications
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          RELEASE_SLACK_CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           METRICS_CUSTOMER_IO_TOKEN: ${{ secrets.METRICS_CUSTOMER_IO_TOKEN }}
           MAILER_SENDER: ${{ secrets.OAUTH_EMAIL_SENDER }}
           MAILER_USER: ${{ secrets.OAUTH_EMAIL_LOGIN }}
@@ -162,7 +165,7 @@ jobs:
         if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
         run: node ./tools/changelog/index.js
         env:
-          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+          CHANNEL_ID: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           DEPLOYED_URL: ${{ steps.set_info.outputs.deployed_url }}
           PREV_VERSION: ${{ needs.output-prev-version.outputs.prev }}
@@ -178,7 +181,7 @@ jobs:
           method: chat.postMessage
           token: ${{ secrets.SLACK_BOT_TOKEN }}
           payload: |
-            channel: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+            channel: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
             text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Backend deploy failed `${{ github.event.inputs.flavor }}`>"
             blocks:
               - type: section
@@ -193,7 +196,7 @@ jobs:
           token: ${{ secrets.SLACK_BOT_TOKEN }}
           method: chat.postMessage
           payload: |
-            channel: ${{ secrets.RELEASE_SLACK_CHNNEL_ID }}
+            channel: ${{ secrets.RELEASE_SLACK_CHANNEL_ID }}
             text: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Backend deploy cancelled `${{ github.event.inputs.flavor }}`>"
             blocks:
               - type: section

packages/backend/server/package.json

@@ -16,6 +16,7 @@
     "test:coverage": "c8 ava --concurrency 1 --serial",
     "test:copilot:e2e:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.e2e.ts\"",
     "test:copilot:spec:coverage": "c8 ava --timeout=5m \"src/__tests__/**/copilot-*.spec.ts\"",
+    "test:copilot:e2e:cron": "node ./scripts/copilot-cron-test.js",
     "data-migration": "cross-env NODE_ENV=script r ./src/data/index.ts",
     "predeploy": "yarn prisma migrate deploy && NODE_ENV=script node --import ./scripts/register.js ./dist/data/index.js run",
     "postinstall": "prisma generate"
@@ -58,7 +59,9 @@
     "@opentelemetry/semantic-conventions": "^1.28.0",
     "@prisma/client": "^5.22.0",
     "@prisma/instrumentation": "^5.22.0",
+    "@slack/web-api": "^7.3.4",
     "@socket.io/redis-adapter": "^8.3.0",
+    "ava": "^6.1.2",
     "cookie-parser": "^1.4.7",
     "dotenv": "^16.4.7",
     "eventsource-parser": "^3.0.0",
@@ -71,8 +74,10 @@
     "html-validate": "^9.0.0",
     "ioredis": "^5.4.1",
     "is-mobile": "^5.0.0",
+    "jsx-slack": "^6.1.1",
     "keyv": "^5.2.2",
     "lodash-es": "^4.17.21",
+    "marked": "^15.0.0",
     "mixpanel": "^0.18.0",
     "mustache": "^4.2.0",
     "nanoid": "^5.0.9",
@@ -89,6 +94,8 @@
     "ses": "^1.10.0",
     "socket.io": "^4.8.1",
     "stripe": "^17.4.0",
+    "supertest": "^7.0.0",
+    "tap-parser": "^18.0.0",
     "ts-node": "^10.9.2",
     "typescript": "^5.7.2",
     "winston": "^3.17.0",
@@ -113,12 +120,10 @@
     "@types/on-headers": "^1.0.3",
     "@types/sinon": "^17.0.3",
     "@types/supertest": "^6.0.2",
-    "ava": "^6.2.0",
     "c8": "^10.1.3",
     "cross-env": "^7.0.3",
     "nodemon": "^3.1.7",
-    "sinon": "^19.0.2",
-    "supertest": "^7.0.0"
+    "sinon": "^19.0.2"
   },
   "ava": {
     "timeout": "1m",

packages/backend/server/scripts/copilot-cron-test.js

@@ -0,0 +1,95 @@
+// start process
+
+import { spawn } from 'node:child_process';
+
+import { WebClient } from '@slack/web-api';
+import { jsxslack } from 'jsx-slack';
+import { marked, Renderer } from 'marked';
+import { Parser } from 'tap-parser';
+
+async function runTest() {
+  const tester = new Promise(resolve => {
+    const test = spawn(
+      'npx',
+      [
+        'ava',
+        '--config',
+        'tests/ava.docker.config.js',
+        'tests/**/copilot-*.e2e.ts',
+        '--tap',
+      ],
+      { env: { ...process.env, NODE_NO_WARNINGS: 1 } }
+    );
+
+    const parser = new Parser();
+    test.stdout.on('data', data => {
+      console.log(data.toString());
+      parser.write(data);
+    });
+
+    test.on('close', _ => {
+      const failures = parser?.failures.filter(f => !!f.fullname);
+      const timeouts = parser?.failures.filter(f => !f.fullname);
+      const result = [
+        `${parser.results.pass} passed`,
+        `${parser.results.fail - timeouts.length} failed`,
+        `${timeouts.length} timeouts`,
+        `${parser.results.skip} skipped`,
+      ];
+      const report = [
+        `Test finished with ${result.join(', ')}.`,
+        failures?.length > 0
+          ? `Failed tests: \n\n${failures.map(failure => `- ${failure.fullname}`).join('\n')}`
+          : '',
+      ];
+      resolve(report.join('\n\n'));
+    });
+  });
+
+  try {
+    return await tester;
+  } catch (e) {
+    return e.message;
+  }
+}
+
+function render(markdown) {
+  const rendered = marked(markdown, {
+    renderer: new (class CustomRenderer extends Renderer {
+      heading({ tokens }) {
+        return `
+            <Fragment>
+              <Section><b>${tokens[0].text}</b></Section>
+              <Divider />
+            </Fragment>`;
+      }
+
+      paragraph({ tokens }) {
+        return `<Section><p>${tokens[0].text}</p></Section>`;
+      }
+
+      list(token) {
+        return `<Section>${super.list(token)}</Section>`;
+      }
+
+      hr() {
+        return `<Divider />`;
+      }
+    })(),
+  });
+  return jsxslack([`<Blocks>${rendered}</Blocks>`]);
+}
+
+const { CHANNEL_ID, SLACK_BOT_TOKEN, AFFINE_ENV } = process.env;
+
+const report = await runTest();
+const blocks = render(
+  [`# AFFiNE Copilot Test ${AFFINE_ENV} Env Test Result`, report].join('\n\n')
+);
+const { ok } = await new WebClient(SLACK_BOT_TOKEN).chat.postMessage({
+  channel: CHANNEL_ID,
+  text: `AFFiNE Copilot Test ${AFFINE_ENV} Env Test Result`,
+  blocks,
+});
+
+console.assert(ok, 'Failed to send a message to Slack');

packages/backend/server/src/__tests__/ava.docker.config.js

@@ -0,0 +1,15 @@
+import packageJson from '../package.json' with { type: 'json' };
+
+export default {
+  ...packageJson.ava,
+  nodeArguments: [
+    '--trace-sigint',
+    '--loader',
+    'ts-node/esm/transpile-only.mjs',
+    '--es-module-specifier-resolution=node',
+  ],
+  environmentVariables: {
+    ...packageJson.ava.environmentVariables,
+    TS_NODE_PROJECT: './tests/tsconfig.docker.json',
+  },
+};