Concurrent frontend code generate

Sma1lboy · Jan 19, 2025 · 940119c · 940119c
1 parent be2aa22
commit 940119c
Show file tree

Hide file tree

Showing 4 changed files with 303 additions and 113 deletions.
diff --git a/backend/src/build-system/__tests__/fullstack-gen.spec.ts b/backend/src/build-system/__tests__/fullstack-gen.spec.ts
@@ -43,9 +43,22 @@ import { BackendFileReviewHandler } from '../handlers/backend/file-review/file-r
           // requires: ['op:UX:SMD'],
         },
         {
-          handler: UXDMDHandler,
-          name: 'UX DataMap Document Node',
-          // requires: ['op:UX:SMD'],
+          handler: DBRequirementHandler,
+          name: 'Database Requirements Node',
+          // requires: ['op:UX:DATAMAP:DOC'],
+        },
+        {
+          handler: FileStructureHandler,
+          name: 'File Structure Generation',
+          // requires: ['op:UX:SMD', 'op:UX:DATAMAP:DOC'],
+          options: {
+            projectPart: 'frontend',
+          },
+        },
+        {
+          handler: UXSMSPageByPageHandler,
+          name: 'Level 2 UX Sitemap Structure Node details',
+          // requires: ['op:UX:SMS'],
         },
         {
           handler: DBRequirementHandler,

diff --git a/backend/src/build-system/handlers/frontend-code-generate/index.ts b/backend/src/build-system/handlers/frontend-code-generate/index.ts
@@ -1,23 +1,24 @@
 import { BuildHandler, BuildResult } from 'src/build-system/types';
 import { BuilderContext } from 'src/build-system/context';
 import { Logger } from '@nestjs/common';
+import { batchChatSyncWithClock } from 'src/build-system/utils/handler-helper';
 import {
-  generateFilesDependency,
   createFile,
+  generateFilesDependencyWithLayers,
 } from '../../utils/file_generator_util';
 import { VirtualDirectory } from '../../virtual-dir';
-import normalizePath from 'normalize-path';
-import * as path from 'path';
-import { readFile } from 'fs/promises';
-
-import { parseGenerateTag } from 'src/build-system/utils/strings';
 
-import { generateFrontEndCodePrompt, generateCSSPrompt } from './prompt';
 import { UXSMSHandler } from '../ux/sitemap-structure';
 import { UXDMDHandler } from '../ux/datamap';
 import { BackendRequirementHandler } from '../backend/requirements-document';
 import { FileFAHandler } from '../file-manager/file-arch';
 import { BuildNode, BuildNodeRequire } from 'src/build-system/hanlder-manager';
+import normalizePath from 'normalize-path';
+import path from 'path';
+import { readFile } from 'fs-extra';
+import { generateCSSPrompt, generateFrontEndCodePrompt } from './prompt';
+import { parseGenerateTag } from 'src/build-system/utils/strings';
+import { ResponseParsingError } from 'src/build-system/errors';
 
 /**
  * FrontendCodeHandler is responsible for generating the frontend codebase
@@ -56,111 +57,159 @@ export class FrontendCodeHandler implements BuildHandler<string> {
     this.virtualDir = context.virtualDirectory;
     const frontendPath = context.getGlobalContext('frontendPath');
 
+    if (
+      !sitemapStruct ||
+      !uxDataMapDoc ||
+      !backendRequirementDoc ||
+      !fileArchDoc
+    ) {
+      this.logger.error(sitemapStruct);
+      this.logger.error(uxDataMapDoc);
+      this.logger.error(backendRequirementDoc);
+      this.logger.error(fileArchDoc);
+      throw new Error('Missing required parameters.');
+    }
+
     // Dependency
-    const { sortedFiles, fileInfos } = await generateFilesDependency(
-      fileArchDoc,
-      this.virtualDir,
-    );
+    const { concurrencyLayers, fileInfos } =
+      await generateFilesDependencyWithLayers(fileArchDoc, this.virtualDir);
 
-    // Iterate the sortedFiles
-    for (const file of sortedFiles) {
-      const currentFullFilePath = normalizePath(
-        path.resolve(frontendPath, 'src', file),
+    // 4. Process each "layer" in sequence; files in a layer in parallel
+    for (const [layerIndex, layer] of concurrencyLayers.entries()) {
+      this.logger.log(
+        `\n==== Concurrency Layer #${layerIndex + 1} ====\nFiles: [${layer.join(
+          ', ',
+        )}]\n`,
       );
 
-      const extension = currentFullFilePath.split('.').pop() || '';
-
-      // Retrieve the direct dependencies for this file
-      const directDepsArray = fileInfos[file]?.dependsOn || [];
-
-      //gather the contents of each dependency into a single string.
-      let dependenciesContext = '';
-      for (const dep of directDepsArray) {
-        try {
-          // Resolve against frontendPath to get the absolute path
-          const resolvedDepPath = normalizePath(
-            path.resolve(frontendPath, 'src', dep),
+      await Promise.all(
+        layer.map(async (file) => {
+          this.logger.log(
+            `Layer #${layerIndex + 1}, generating code for file: ${file}`,
           );
 
-          // Read the file. (may want to guard so only read certain file types.)
-          const fileContent = await readFile(resolvedDepPath, 'utf-8');
-
-          //just append a code:
-          dependenciesContext += `\n\n[Dependency: ${dep}]\n\`\`\`\n${fileContent}\n\`\`\`\n`;
-        } catch (readError) {
-          // If the file doesn't exist or can't be read, log a warning.
-          this.logger.warn(
-            `Failed to read dependency "${dep}" for file "${file}": ${readError}`,
+          // Resolve the absolute path where this file should be generated
+          const currentFullFilePath = normalizePath(
+            path.resolve(frontendPath, 'src', file),
           );
-        }
-      }
 
-      // Format for the prompt
-      const directDependencies = directDepsArray.join('\n');
+          // Gather direct dependencies
+          const directDepsArray = fileInfos[file]?.dependsOn || [];
+          const dependenciesContext = '';
+
+          // Read each dependency and append to dependenciesContext
+          let dependenciesText = '';
+          for (const dep of directDepsArray) {
+            try {
+              const resolvedDepPath = normalizePath(
+                path.resolve(frontendPath, 'src', dep),
+              );
+              const depContent = await readFile(resolvedDepPath, 'utf-8');
+              dependenciesText += `\n\nprevious code **${dep}** is:\n\`\`\`typescript\n${depContent}\n\`\`\`\n`;
+            } catch (err) {
+              this.logger.warn(
+                `Failed to read dependency "${dep}" for file "${file}": ${err}`,
+              );
+              throw new ResponseParsingError(
+                `Error generating code for ${file}:`,
+              );
+            }
+          }
+
+          // 5. Build prompt text depending on file extension
+          const fileExtension = path.extname(file);
+          let frontendCodePrompt = '';
+          if (fileExtension === '.css') {
+            frontendCodePrompt = generateCSSPrompt(
+              file,
+              directDepsArray.join('\n'),
+            );
+          } else {
+            // default: treat as e.g. .ts, .js, .vue, .jsx, etc.
+            frontendCodePrompt = generateFrontEndCodePrompt(
+              file,
+              directDepsArray.join('\n'),
+            );
+          }
+          this.logger.log(
+            `Prompt for file "${file}":\n${frontendCodePrompt}\n`,
+          );
 
-      this.logger.log(
-        `Generating file in dependency order: ${currentFullFilePath}`,
-      );
-      this.logger.log(
-        `2 Generating file in dependency order directDependencies: ${directDependencies}`,
+          const messages = [
+            {
+              role: 'system' as const,
+              content: frontendCodePrompt,
+            },
+            {
+              role: 'user' as const,
+              content: `This is the Sitemap Structure:
+              ${sitemapStruct}
+              
+              Next will provide Sitemap Structure.`,
+            },
+            {
+              role: 'user' as const,
+              content: `This is the UX Datamap Documentation:
+              ${uxDataMapDoc}
+              
+              Next will provide UX Datamap Documentation.`,
+            },
+            {
+              role: 'user' as const,
+              content: `This is the Backend Requirement Documentation:
+              ${backendRequirementDoc}
+              
+              Next will provide Backend Requirement Documentation.`,
+            },
+
+            {
+              role: 'user' as const,
+              content: `Dependencies for ${file}:\n${dependenciesText}\n
+
+            Now generate code for "${file}".`,
+            },
+          ];
+
+          // 6. Call your Chat Model
+          let generatedCode = '';
+          try {
+            const modelResponse = await batchChatSyncWithClock(
+              context,
+              'generate frontend code',
+              FrontendCodeHandler.name,
+              [
+                {
+                  model: 'gpt-4o',
+                  messages,
+                },
+              ],
+            );
+
+            generatedCode = parseGenerateTag(modelResponse[0]);
+          } catch (err) {
+            this.logger.error(`Error generating code for ${file}:`, err);
+            throw new ResponseParsingError(
+              `Error generating code for ${file}:`,
+            );
+          }
+
+          // 7. Write the file to the filesystem
+          await createFile(currentFullFilePath, generatedCode);
+
+          this.logger.log(
+            `Layer #${layerIndex + 1}, completed generation for file: ${file}`,
+          );
+        }),
       );
 
-      let frontendCodePrompt = '';
-
-      if (extension === 'css') {
-        frontendCodePrompt = generateCSSPrompt(
-          sitemapStruct,
-          uxDataMapDoc,
-          file,
-          directDependencies,
-          dependenciesContext,
-        );
-      } else {
-        // Generate the prompt
-        frontendCodePrompt = generateFrontEndCodePrompt(
-          sitemapStruct,
-          uxDataMapDoc,
-          backendRequirementDoc.overview,
-          file,
-          directDependencies,
-          dependenciesContext,
-        );
-      }
       this.logger.log(
-        'generate code prompt for frontendCodePrompt or css: ' +
-          frontendCodePrompt,
+        `\n==== Finished concurrency layer #${layerIndex + 1} ====\n`,
       );
-
-      this.logger.debug('Generated frontend code prompt.');
-
-      let generatedCode = '';
-      const model = 'gpt-4o-mini';
-      try {
-        // Call the model
-        const modelResponse = await context.model.chatSync({
-          model,
-          messages: [{ content: frontendCodePrompt, role: 'system' }],
-        });
-
-        // Parse the output
-        generatedCode = parseGenerateTag(modelResponse);
-
-        this.logger.debug('Frontend code generated and parsed successfully.');
-      } catch (error) {
-        // Return error
-        this.logger.error('Error during frontend code generation:', error);
-        return {
-          success: false,
-          error: new Error('Failed to generate frontend code.'),
-        };
-      }
-
-      await createFile(currentFullFilePath, generatedCode);
     }
 
     return {
       success: true,
-      data: 'test',
+      data: frontendPath,
       error: new Error('Frontend code generated and parsed successfully.'),
     };
   }

diff --git a/backend/src/build-system/handlers/frontend-code-generate/prompt.ts b/backend/src/build-system/handlers/frontend-code-generate/prompt.ts
@@ -1,23 +1,19 @@
 export const generateFrontEndCodePrompt = (
-  sitemapStruct: string,
-  uxDatamapDoc: string,
-  backendRequirementDoc: string,
   currentFile: string,
   dependencyFilePath: string,
-  dependenciesContext: string,
 ): string => {
   return `You are an expert frontend developer. 
   Your task is to generate complete and production-ready React frontend code based on the provided inputs using typescript.
   The code should include all necessary files, folders, and logic to cover UI components, API integration, routing, and state management while ensuring scalability and maintainability.
   
   Based on following inputs:
 
-   - Sitemap Structure: ${sitemapStruct}
-   - UX Datamap Documentation: ${uxDatamapDoc}
-   - Backend Requirement Documentation: ${backendRequirementDoc}
+   - Sitemap Structure: 
+   - UX Datamap Documentation: 
+   - Backend Requirement Documentation: 
    - Current File: ${currentFile}
    - dependencyFilePath: ${dependencyFilePath}
-   - Dependency File: ${dependenciesContext}
+   - Dependency File Code: 
 
     ### Instructions and Rules:
         File Requirements:
@@ -62,24 +58,20 @@ export const generateFrontEndCodePrompt = (
 };
 
 export function generateCSSPrompt(
-  sitemapStruct: string,
-  uxDatamapDoc: string,
   fileName: string,
   directDependencies: string,
-  dependenciesContext: string,
 ): string {
   return `
   You are an expert CSS developer. Generate valid, production-ready CSS for the file "${fileName}".
 
     ## Context
-    - Sitemap Strucutrue: ${sitemapStruct}
-    - UX Datamap Documentation: ${uxDatamapDoc}
+    - Sitemap Strucutrue: 
+    - UX Datamap Documentation: 
 
     - Direct Dependencies (if any and may include references to other styles or partials):
     ${directDependencies}
 
-    - Direct Dependencies Context (if any):
-    ${dependenciesContext}
+    - Direct Dependencies Context:
 
   ## Rules & Guidelines
     1. **Do NOT** include any JavaScript or React code—only plain CSS.