diff --git a/backend/src/build-system/__tests__/fullstack-gen.spec.ts b/backend/src/build-system/__tests__/fullstack-gen.spec.ts index 8e011ef..5ee5599 100644 --- a/backend/src/build-system/__tests__/fullstack-gen.spec.ts +++ b/backend/src/build-system/__tests__/fullstack-gen.spec.ts @@ -43,9 +43,22 @@ import { BackendFileReviewHandler } from '../handlers/backend/file-review/file-r // requires: ['op:UX:SMD'], }, { - handler: UXDMDHandler, - name: 'UX DataMap Document Node', - // requires: ['op:UX:SMD'], + handler: DBRequirementHandler, + name: 'Database Requirements Node', + // requires: ['op:UX:DATAMAP:DOC'], + }, + { + handler: FileStructureHandler, + name: 'File Structure Generation', + // requires: ['op:UX:SMD', 'op:UX:DATAMAP:DOC'], + options: { + projectPart: 'frontend', + }, + }, + { + handler: UXSMSPageByPageHandler, + name: 'Level 2 UX Sitemap Structure Node details', + // requires: ['op:UX:SMS'], }, { handler: DBRequirementHandler, diff --git a/backend/src/build-system/handlers/frontend-code-generate/index.ts b/backend/src/build-system/handlers/frontend-code-generate/index.ts index 8eb6a1a..e271f28 100644 --- a/backend/src/build-system/handlers/frontend-code-generate/index.ts +++ b/backend/src/build-system/handlers/frontend-code-generate/index.ts @@ -1,23 +1,24 @@ import { BuildHandler, BuildResult } from 'src/build-system/types'; import { BuilderContext } from 'src/build-system/context'; import { Logger } from '@nestjs/common'; +import { batchChatSyncWithClock } from 'src/build-system/utils/handler-helper'; import { - generateFilesDependency, createFile, + generateFilesDependencyWithLayers, } from '../../utils/file_generator_util'; import { VirtualDirectory } from '../../virtual-dir'; -import normalizePath from 'normalize-path'; -import * as path from 'path'; -import { readFile } from 'fs/promises'; - -import { parseGenerateTag } from 'src/build-system/utils/strings'; -import { generateFrontEndCodePrompt, generateCSSPrompt } from './prompt'; import { UXSMSHandler } from '../ux/sitemap-structure'; import { UXDMDHandler } from '../ux/datamap'; import { BackendRequirementHandler } from '../backend/requirements-document'; import { FileFAHandler } from '../file-manager/file-arch'; import { BuildNode, BuildNodeRequire } from 'src/build-system/hanlder-manager'; +import normalizePath from 'normalize-path'; +import path from 'path'; +import { readFile } from 'fs-extra'; +import { generateCSSPrompt, generateFrontEndCodePrompt } from './prompt'; +import { parseGenerateTag } from 'src/build-system/utils/strings'; +import { ResponseParsingError } from 'src/build-system/errors'; /** * FrontendCodeHandler is responsible for generating the frontend codebase @@ -56,111 +57,159 @@ export class FrontendCodeHandler implements BuildHandler { this.virtualDir = context.virtualDirectory; const frontendPath = context.getGlobalContext('frontendPath'); + if ( + !sitemapStruct || + !uxDataMapDoc || + !backendRequirementDoc || + !fileArchDoc + ) { + this.logger.error(sitemapStruct); + this.logger.error(uxDataMapDoc); + this.logger.error(backendRequirementDoc); + this.logger.error(fileArchDoc); + throw new Error('Missing required parameters.'); + } + // Dependency - const { sortedFiles, fileInfos } = await generateFilesDependency( - fileArchDoc, - this.virtualDir, - ); + const { concurrencyLayers, fileInfos } = + await generateFilesDependencyWithLayers(fileArchDoc, this.virtualDir); - // Iterate the sortedFiles - for (const file of sortedFiles) { - const currentFullFilePath = normalizePath( - path.resolve(frontendPath, 'src', file), + // 4. Process each "layer" in sequence; files in a layer in parallel + for (const [layerIndex, layer] of concurrencyLayers.entries()) { + this.logger.log( + `\n==== Concurrency Layer #${layerIndex + 1} ====\nFiles: [${layer.join( + ', ', + )}]\n`, ); - const extension = currentFullFilePath.split('.').pop() || ''; - - // Retrieve the direct dependencies for this file - const directDepsArray = fileInfos[file]?.dependsOn || []; - - //gather the contents of each dependency into a single string. - let dependenciesContext = ''; - for (const dep of directDepsArray) { - try { - // Resolve against frontendPath to get the absolute path - const resolvedDepPath = normalizePath( - path.resolve(frontendPath, 'src', dep), + await Promise.all( + layer.map(async (file) => { + this.logger.log( + `Layer #${layerIndex + 1}, generating code for file: ${file}`, ); - // Read the file. (may want to guard so only read certain file types.) - const fileContent = await readFile(resolvedDepPath, 'utf-8'); - - //just append a code: - dependenciesContext += `\n\n[Dependency: ${dep}]\n\`\`\`\n${fileContent}\n\`\`\`\n`; - } catch (readError) { - // If the file doesn't exist or can't be read, log a warning. - this.logger.warn( - `Failed to read dependency "${dep}" for file "${file}": ${readError}`, + // Resolve the absolute path where this file should be generated + const currentFullFilePath = normalizePath( + path.resolve(frontendPath, 'src', file), ); - } - } - // Format for the prompt - const directDependencies = directDepsArray.join('\n'); + // Gather direct dependencies + const directDepsArray = fileInfos[file]?.dependsOn || []; + const dependenciesContext = ''; + + // Read each dependency and append to dependenciesContext + let dependenciesText = ''; + for (const dep of directDepsArray) { + try { + const resolvedDepPath = normalizePath( + path.resolve(frontendPath, 'src', dep), + ); + const depContent = await readFile(resolvedDepPath, 'utf-8'); + dependenciesText += `\n\nprevious code **${dep}** is:\n\`\`\`typescript\n${depContent}\n\`\`\`\n`; + } catch (err) { + this.logger.warn( + `Failed to read dependency "${dep}" for file "${file}": ${err}`, + ); + throw new ResponseParsingError( + `Error generating code for ${file}:`, + ); + } + } + + // 5. Build prompt text depending on file extension + const fileExtension = path.extname(file); + let frontendCodePrompt = ''; + if (fileExtension === '.css') { + frontendCodePrompt = generateCSSPrompt( + file, + directDepsArray.join('\n'), + ); + } else { + // default: treat as e.g. .ts, .js, .vue, .jsx, etc. + frontendCodePrompt = generateFrontEndCodePrompt( + file, + directDepsArray.join('\n'), + ); + } + this.logger.log( + `Prompt for file "${file}":\n${frontendCodePrompt}\n`, + ); - this.logger.log( - `Generating file in dependency order: ${currentFullFilePath}`, - ); - this.logger.log( - `2 Generating file in dependency order directDependencies: ${directDependencies}`, + const messages = [ + { + role: 'system' as const, + content: frontendCodePrompt, + }, + { + role: 'user' as const, + content: `This is the Sitemap Structure: + ${sitemapStruct} + + Next will provide Sitemap Structure.`, + }, + { + role: 'user' as const, + content: `This is the UX Datamap Documentation: + ${uxDataMapDoc} + + Next will provide UX Datamap Documentation.`, + }, + { + role: 'user' as const, + content: `This is the Backend Requirement Documentation: + ${backendRequirementDoc} + + Next will provide Backend Requirement Documentation.`, + }, + + { + role: 'user' as const, + content: `Dependencies for ${file}:\n${dependenciesText}\n + + Now generate code for "${file}".`, + }, + ]; + + // 6. Call your Chat Model + let generatedCode = ''; + try { + const modelResponse = await batchChatSyncWithClock( + context, + 'generate frontend code', + FrontendCodeHandler.name, + [ + { + model: 'gpt-4o', + messages, + }, + ], + ); + + generatedCode = parseGenerateTag(modelResponse[0]); + } catch (err) { + this.logger.error(`Error generating code for ${file}:`, err); + throw new ResponseParsingError( + `Error generating code for ${file}:`, + ); + } + + // 7. Write the file to the filesystem + await createFile(currentFullFilePath, generatedCode); + + this.logger.log( + `Layer #${layerIndex + 1}, completed generation for file: ${file}`, + ); + }), ); - let frontendCodePrompt = ''; - - if (extension === 'css') { - frontendCodePrompt = generateCSSPrompt( - sitemapStruct, - uxDataMapDoc, - file, - directDependencies, - dependenciesContext, - ); - } else { - // Generate the prompt - frontendCodePrompt = generateFrontEndCodePrompt( - sitemapStruct, - uxDataMapDoc, - backendRequirementDoc.overview, - file, - directDependencies, - dependenciesContext, - ); - } this.logger.log( - 'generate code prompt for frontendCodePrompt or css: ' + - frontendCodePrompt, + `\n==== Finished concurrency layer #${layerIndex + 1} ====\n`, ); - - this.logger.debug('Generated frontend code prompt.'); - - let generatedCode = ''; - const model = 'gpt-4o-mini'; - try { - // Call the model - const modelResponse = await context.model.chatSync({ - model, - messages: [{ content: frontendCodePrompt, role: 'system' }], - }); - - // Parse the output - generatedCode = parseGenerateTag(modelResponse); - - this.logger.debug('Frontend code generated and parsed successfully.'); - } catch (error) { - // Return error - this.logger.error('Error during frontend code generation:', error); - return { - success: false, - error: new Error('Failed to generate frontend code.'), - }; - } - - await createFile(currentFullFilePath, generatedCode); } return { success: true, - data: 'test', + data: frontendPath, error: new Error('Frontend code generated and parsed successfully.'), }; } diff --git a/backend/src/build-system/handlers/frontend-code-generate/prompt.ts b/backend/src/build-system/handlers/frontend-code-generate/prompt.ts index b4cb7ba..eafd639 100644 --- a/backend/src/build-system/handlers/frontend-code-generate/prompt.ts +++ b/backend/src/build-system/handlers/frontend-code-generate/prompt.ts @@ -1,10 +1,6 @@ export const generateFrontEndCodePrompt = ( - sitemapStruct: string, - uxDatamapDoc: string, - backendRequirementDoc: string, currentFile: string, dependencyFilePath: string, - dependenciesContext: string, ): string => { return `You are an expert frontend developer. Your task is to generate complete and production-ready React frontend code based on the provided inputs using typescript. @@ -12,12 +8,12 @@ export const generateFrontEndCodePrompt = ( Based on following inputs: - - Sitemap Structure: ${sitemapStruct} - - UX Datamap Documentation: ${uxDatamapDoc} - - Backend Requirement Documentation: ${backendRequirementDoc} + - Sitemap Structure: + - UX Datamap Documentation: + - Backend Requirement Documentation: - Current File: ${currentFile} - dependencyFilePath: ${dependencyFilePath} - - Dependency File: ${dependenciesContext} + - Dependency File Code: ### Instructions and Rules: File Requirements: @@ -62,24 +58,20 @@ export const generateFrontEndCodePrompt = ( }; export function generateCSSPrompt( - sitemapStruct: string, - uxDatamapDoc: string, fileName: string, directDependencies: string, - dependenciesContext: string, ): string { return ` You are an expert CSS developer. Generate valid, production-ready CSS for the file "${fileName}". ## Context - - Sitemap Strucutrue: ${sitemapStruct} - - UX Datamap Documentation: ${uxDatamapDoc} + - Sitemap Strucutrue: + - UX Datamap Documentation: - Direct Dependencies (if any and may include references to other styles or partials): ${directDependencies} - - Direct Dependencies Context (if any): - ${dependenciesContext} + - Direct Dependencies Context: ## Rules & Guidelines 1. **Do NOT** include any JavaScript or React code—only plain CSS. diff --git a/backend/src/build-system/utils/file_generator_util.ts b/backend/src/build-system/utils/file_generator_util.ts index 759b823..58dd83c 100644 --- a/backend/src/build-system/utils/file_generator_util.ts +++ b/backend/src/build-system/utils/file_generator_util.ts @@ -15,8 +15,70 @@ interface GenerateFilesDependencyResult { fileInfos: Record; } +interface GenerateFilesDependencyLayerResult { + concurrencyLayers: string[][]; + fileInfos: Record; +} + const logger = new Logger('FileGeneratorUtil'); +export async function generateFilesDependencyWithLayers( + jsonString: string, + virtualDirectory: VirtualDirectory, +): Promise { + // 1. Parse for JSON + const jsonData = JSON.parse(jsonString); + + // 2. Build a "fileInfos" object with .dependsOn + const { fileInfos, nodes } = buildDependencyLayerGraph(jsonData); + + // 3. Validate the files actually exist in VirtualDirectory + validateAgainstVirtualDirectory(nodes, virtualDirectory); + + // 4. Build concurrency layers with Kahn’s Algorithm + const concurrencyLayers = buildConcurrencyLayers(nodes, fileInfos); + + // Optionally check for cycles (if you didn’t do it inside buildConcurrencyLayers) + // detectCycles(...) or similar + + logger.log('All files dependency layers generated successfully.'); + + return { + concurrencyLayers, + fileInfos, + }; +} + +function buildDependencyLayerGraph(jsonData: { + files: Record; +}): { + fileInfos: Record; + nodes: Set; +} { + const fileInfos: Record = {}; + const nodes = new Set(); + + Object.entries(jsonData.files).forEach(([fileName, details]) => { + nodes.add(fileName); + + // Initialize the record + fileInfos[fileName] = { + filePath: fileName, + dependsOn: [], + }; + + // In the JSON, "dependsOn" is an array of file paths + details.dependsOn.forEach((dep) => { + const resolvedDep = resolveDependency(fileName, dep); + nodes.add(resolvedDep); + + fileInfos[fileName].dependsOn.push(resolvedDep); + }); + }); + + return { fileInfos, nodes }; +} + /** * Generates files based on JSON extracted from a Markdown document. * Ensures dependency order is maintained during file creation. @@ -173,3 +235,77 @@ export async function createFile( logger.log(`File created: ${filePath}`); } + +/** + * Creates concurrency layers (or "waves") of files so that + * files with no remaining dependencies can be processed in parallel. + * + * @param nodes - The set of all files in your project. + * @param fileInfos - A record of each file and its direct dependencies. + * @returns An array of arrays, where each sub-array is a concurrency layer. + */ +function buildConcurrencyLayers( + nodes: Set, + fileInfos: Record, +): string[][] { + // 1. Compute in-degrees: how many dependencies each file has + const inDegree: Record = {}; + for (const file of nodes) { + inDegree[file] = 0; + } + + // For each file, increment the in-degree of the file that depends on it + // In fileInfos, "fileInfos[child].dependsOn = [dep1, dep2...]" + // means edges: dep1 -> child, dep2 -> child, etc. + // So the child’s in-degree = # of dependencies + for (const child of nodes) { + const deps = fileInfos[child]?.dependsOn || []; + for (const dep of deps) { + inDegree[child] = (inDegree[child] ?? 0) + 1; + } + } + + // 2. Collect the initial layer: all files with in-degree == 0 + let layer = Object.entries(inDegree) + .filter(([_, deg]) => deg === 0) + .map(([file]) => file); + + const resultLayers: string[][] = []; + + // 3. Build each layer until no more zero in-degree nodes remain + while (layer.length > 0) { + resultLayers.push(layer); + + // We'll build the next layer by removing all the current layer’s + // edges from the graph. + const nextLayer: string[] = []; + + // For each file in the current layer + for (const file of layer) { + // Find "children" (which are the files that depend on `file`) + for (const possibleChild of nodes) { + if (fileInfos[possibleChild]?.dependsOn?.includes(file)) { + // Decrement the child's in-degree + inDegree[possibleChild]--; + if (inDegree[possibleChild] === 0) { + nextLayer.push(possibleChild); + } + } + } + } + + layer = nextLayer; + } + + // 4. If there are any files left with in-degree > 0, there's a cycle + const unprocessed = Object.entries(inDegree).filter(([_, deg]) => deg > 0); + if (unprocessed.length > 0) { + throw new Error( + `Cycle or leftover dependencies detected for: ${unprocessed + .map(([f]) => f) + .join(', ')}`, + ); + } + + return resultLayers; +}