feat(backend): backend error handling strategy

backend/src/build-system/__tests__/test.fullstack-gen.spec.ts

@@ -1,8 +1,18 @@
 import { BuildSequence } from '../types';
-import { executeBuildSequence, objectToMarkdown, writeToFile } from './utils';
+import * as fs from 'fs';
+import * as path from 'path';
+import { writeToFile } from './utils';
+import { BuildMonitor } from '../monitor';
+import { BuilderContext } from '../context';
 
-describe('Build Sequence Test', () => {
-  it('should execute build sequence successfully', async () => {
+describe('Sequence: PRD -> UXSD -> UXSS -> UXDD -> DATABASE_REQ -> DBSchemas -> Frontend_File_struct -> Frontend_File_arch -> BackendCodeGenerator', () => {
+  // Generate a unique folder with a timestamp
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const logFolderPath = `./logs/Fullstack_code_generator-${timestamp}`;
+  fs.mkdirSync(logFolderPath, { recursive: true });
+
+  it('should execute the frontend and backend code generation sequence and log results to individual files', async () => {
+    // Define the build sequence up to Backend Code Generator
     const sequence: BuildSequence = {
       id: 'test-backend-sequence',
       version: '1.0.0',
@@ -139,10 +149,104 @@ describe('Build Sequence Test', () => {
         },
       ],
     };
+    const context = new BuilderContext(sequence, 'test-env');
+    const monitor = BuildMonitor.getInstance();
+
+    try {
+      console.time('Total Execution Time');
+
+      await context.execute();
+
+      console.timeEnd('Total Execution Time');
+
+      const monitorReport = monitor.generateTextReport(sequence.id);
+      fs.writeFileSync(
+        path.join(logFolderPath, 'execution-metrics.txt'),
+        monitorReport,
+        'utf8',
+      );
+
+      const sequenceMetrics = monitor.getSequenceMetrics(sequence.id);
+      if (sequenceMetrics) {
+        const metricsJson = {
+          totalDuration: `${sequenceMetrics.duration}ms`,
+          successRate: `${sequenceMetrics.successRate.toFixed(2)}%`,
+          totalSteps: sequenceMetrics.totalSteps,
+          completedSteps: sequenceMetrics.completedSteps,
+          failedSteps: sequenceMetrics.failedSteps,
+          totalNodes: sequenceMetrics.totalNodes,
+          startTime: new Date(sequenceMetrics.startTime).toISOString(),
+          endTime: new Date(sequenceMetrics.endTime).toISOString(),
+        };
+
+        fs.writeFileSync(
+          path.join(logFolderPath, 'metrics.json'),
+          JSON.stringify(metricsJson, null, 2),
+          'utf8',
+        );
+
+        console.log('\nSequence Metrics:');
+        console.table(metricsJson);
+      }
+
+      for (const step of sequence.steps) {
+        const stepMetrics = sequenceMetrics?.stepMetrics.get(step.id);
+        for (const node of step.nodes) {
+          const resultData = await context.getNodeData(node.id);
+          const nodeMetrics = stepMetrics?.nodeMetrics.get(node.id);
+          if (resultData) {
+            writeToFile(logFolderPath, `${node.name}`, resultData);
+          } else {
+            console.error(
+              `  Error: Handler ${node.name} failed to produce result data`,
+            );
+            writeToFile(logFolderPath, `${node.name}-error`, {
+              error: 'No result data',
+              metrics: nodeMetrics,
+            });
+          }
+        }
+      }
+
+      const summary = {
+        timestamp: new Date().toISOString(),
+        sequenceId: sequence.id,
+        sequenceName: sequence.name,
+        totalExecutionTime: `${sequenceMetrics?.duration}ms`,
+        successRate: `${sequenceMetrics?.successRate.toFixed(2)}%`,
+        nodesExecuted: sequenceMetrics?.totalNodes,
+        completedNodes: sequenceMetrics?.stepMetrics.size,
+        logFolder: logFolderPath,
+      };
+
+      fs.writeFileSync(
+        path.join(logFolderPath, 'execution-summary.json'),
+        JSON.stringify(summary, null, 2),
+        'utf8',
+      );
+    } catch (error) {
+      const errorReport = {
+        error: {
+          message: error.message,
+          stack: error.stack,
+        },
+        metrics: monitor.getSequenceMetrics(sequence.id),
+        timestamp: new Date().toISOString(),
+      };
+
+      fs.writeFileSync(
+        path.join(logFolderPath, 'error-with-metrics.json'),
+        JSON.stringify(errorReport, null, 2),
+        'utf8',
+      );
 
-    const result = await executeBuildSequence('fullstack-code-gen', sequence);
-    expect(result.success).toBe(true);
-    expect(result.metrics).toBeDefined();
-    console.log(`Logs saved to: ${result.logFolderPath}`);
-  }, 300000);
+      console.error('\nError during sequence execution:');
+      console.error(error);
+      console.error(
+        '\nError report saved to:',
+        path.join(logFolderPath, 'error-with-metrics.json'),
+      );
+      throw new Error('Sequence execution failed.');
+    }
+  }, 300000); // Timeout set to 10 minutes
 });

backend/src/build-system/context.ts

@@ -1,5 +1,6 @@
 import {
   BuildExecutionState,
+  BuildHandler,
   BuildNode,
   BuildResult,
   BuildSequence,
@@ -12,6 +13,7 @@ import { ModelProvider } from 'src/common/model-provider';
 import { v4 as uuidv4 } from 'uuid';
 import { BuildMonitor } from './monitor';
 import { BuildHandlerManager } from './hanlder-manager';
+import { RetryHandler } from './retry-handler';
 
 /**
  * Global data keys used throughout the build process
@@ -50,11 +52,13 @@ export class BuilderContext {
     waiting: new Set(),
   };
 
+  private globalPromises: Set<Promise<any>> = new Set();
   private logger: Logger;
   private globalContext: Map<GlobalDataKeys | string, any> = new Map();
   private nodeData: Map<string, any> = new Map();
 
   private handlerManager: BuildHandlerManager;
+  private retryHandler: RetryHandler;
   private monitor: BuildMonitor;
   public model: ModelProvider;
   public virtualDirectory: VirtualDirectory;
@@ -63,6 +67,7 @@ export class BuilderContext {
     private sequence: BuildSequence,
     id: string,
   ) {
+    this.retryHandler = RetryHandler.getInstance();
     this.handlerManager = BuildHandlerManager.getInstance();
     this.model = ModelProvider.getInstance();
     this.monitor = BuildMonitor.getInstance();
@@ -157,7 +162,7 @@ export class BuilderContext {
           const batch = executableNodes.slice(i, i + concurrencyLimit);
 
           try {
-            const nodeExecutionPromises = batch.map(async (node) => {
+            batch.map(async (node) => {
               if (this.executionState.completed.has(node.id)) {
                 return;
               }
@@ -169,6 +174,7 @@ export class BuilderContext {
                 currentStep.id,
               );
 
+              let res;
               try {
                 if (!this.canExecute(node.id)) {
                   this.logger.log(
@@ -185,7 +191,8 @@ export class BuilderContext {
                 }
 
                 this.logger.log(`Executing node ${node.id} in parallel batch`);
-                await this.executeNodeById(node.id);
+                res = this.executeNodeById(node.id);
+                this.globalPromises.add(res);
 
                 this.monitor.endNodeExecution(
                   node.id,
@@ -205,8 +212,7 @@ export class BuilderContext {
               }
             });
 
-            await Promise.all(nodeExecutionPromises);
-
+            await Promise.all(this.globalPromises);
             const activeModelPromises = this.model.getAllActivePromises();
             if (activeModelPromises.length > 0) {
               this.logger.debug(
@@ -336,7 +342,7 @@ export class BuilderContext {
       this.executionState.completed.has(nodeId) ||
       this.executionState.pending.has(nodeId)
     ) {
-      this.logger.debug(`Node ${nodeId} is already completed or pending.`);
+      //this.logger.debug(`Node ${nodeId} is already completed or pending.`);
       return false;
     }
 
@@ -361,6 +367,7 @@ export class BuilderContext {
       this.executionState.pending.add(nodeId);
       const result = await this.invokeNodeHandler<T>(node);
       this.executionState.completed.add(nodeId);
+      this.logger.log(`${nodeId} is completed`);
       this.executionState.pending.delete(nodeId);
 
       this.nodeData.set(node.id, result.data);
@@ -438,10 +445,23 @@ export class BuilderContext {
 
   private async invokeNodeHandler<T>(node: BuildNode): Promise<BuildResult<T>> {
     const handler = this.handlerManager.getHandler(node.id);
+    this.logger.log(`sovling ${node.id}`);
     if (!handler) {
       throw new Error(`No handler found for node: ${node.id}`);
     }
-
-    return handler.run(this, node.options);
+    try {
+      return await handler.run(this, node.options);
+    } catch (e) {
+      this.logger.error(`retrying ${node.id}`);
+      const result = await this.retryHandler.retryMethod(
+        e,
+        (node) => this.invokeNodeHandler(node),
+        [node],
+      );
+      if (result === undefined) {
+        throw e;
+      }
+      return result as unknown as BuildResult<T>;
+    }
   }
 }

backend/src/build-system/retry-handler.ts

@@ -0,0 +1,59 @@
+import { Logger } from '@nestjs/common';
+
+export class RetryHandler {
+  private readonly logger = new Logger(RetryHandler.name);
+  private static instance: RetryHandler;
+  private retryCounts: Map<string, number> = new Map();
+  public static getInstance() {
+    if (this.instance) return this.instance;
+    return new RetryHandler();
+  }
+  async retryMethod(
+    error: Error,
+    upperMethod: (...args: any[]) => void,
+    args: any[],
+  ): Promise<void> {
+    const errorName = error.name;
+    let retryCount = this.retryCounts.get(errorName) || 0;
+    const isRetrying = this.retryCounts.has(errorName);
+    let res;
+    switch (errorName) {
+      case 'GeneratedTagError':
+      case 'TimeoutError':
+        this.logger.warn(
+          `Retryable error occurred: ${error.message}. Retrying...`,
+        );
+        if (!isRetrying) this.retryCounts.set(errorName, 0);
+        else
+          this.retryCounts.set(errorName, this.retryCounts.get(errorName) + 1);
+        while (retryCount < 3) {
+          try {
+            this.logger.log(`retryCount: ${retryCount}`);
+            res = await upperMethod(...args);
+          } catch (e) {
+            if (e.name === errorName) {
+              retryCount++;
+              this.retryCounts.set(errorName, retryCount);
+              this.logger.warn(
+                `Retryable error occurred: ${e.name}: ${e.message}. Retrying attempt ${retryCount}...`,
+              );
+            } else {
+              this.logger.error(
+                `Non-retryable error occurred: ${e.name}: ${e.message}. Terminating process.`,
+              );
+              throw e;
+            }
+          }
+        }
+        this.retryCounts.delete(errorName);
+        if (res) return res;
+        this.logger.error('Max retry attempts reached. Terminating process.');
+        throw new Error('Max retry attempts reached');
+      default:
+        this.logger.error(
+          `Non-retryable error occurred: ${error.name}: ${error.message}. Terminating process.`,
+        );
+        throw error;
+    }
+  }
+}