HMR Dev Feedback Stats - RsPack

[joeldickson]

The primary goal is to measure and collect data about the developer's real-time feedback loop during local development. Specifically, we want to track how long it takes from when a developer saves a file change until they can see that change reflected in their browser. This "time-to-feedback" metric is crucial for understanding developer experience and identifying potential bottlenecks in the development workflow. By collecting this timing data across different projects and development setups, we can identify patterns, compare build tool performance, and make data-driven decisions about development tooling.

For instance, we can understand if certain types of changes take longer to reflect, if particular project structures impact build times, or how different build tools (like Vite vs Rspack) perform in real-world development scenarios rather than just benchmark tests.

here's he plan for rspack implementation

/**
 * Implementation Plan for Rspack Developer Feedback Plugin
 * 
 * A plugin to measure and collect real-time developer feedback metrics in Rspack,
 * specifically tracking time-to-feedback for file changes during development.
 */

import { Compiler, Compilation } from '@rspack/core';
import WebSocket from 'ws';
import path from 'path';
import { createServer } from 'http';

interface TimingEntry {
  file: string;
  changeDetectedAt: number;
  compileStartAt?: number;
  compileDoneAt?: number;
  hmrAppliedAt?: number;
}

interface MetricsPayload {
  file: string;
  totalDuration: number;
  phases: {
    changeToCompile: number;
    compileDuration: number;
    hmrApplyDuration: number;
  };
  metadata: {
    fileType: string;
    fileSize: number;
    projectType: string;
  };
}

/**
 * Main plugin class for collecting development feedback metrics
 */
class RspackDevFeedbackPlugin {
  private changeMap = new Map<string, TimingEntry>();
  private wsServer: WebSocket.Server;
  private connections: Set<WebSocket> = new Set();
  private metricsBuffer: MetricsPayload[] = [];
  private options: {
    metricsEndpoint?: string;
    logLocally?: boolean;
    anonymizePaths?: boolean;
    flushInterval?: number;
  };

  constructor(options = {}) {
    this.options = {
      logLocally: true,
      anonymizePaths: true,
      flushInterval: 5000,
      ...options
    };
    
    // Initialize WebSocket server for real-time metrics
    const httpServer = createServer();
    this.wsServer = new WebSocket.Server({ server: httpServer });
    httpServer.listen(0, () => {
      const port = (httpServer.address() as any).port;
      console.log(`Metrics WebSocket server listening on port ${port}`);
    });

    this.setupWebSocket();
    this.startMetricsFlush();
  }

  private setupWebSocket() {
    this.wsServer.on('connection', (ws) => {
      this.connections.add(ws);
      ws.on('close', () => this.connections.delete(ws));
    });
  }

  private startMetricsFlush() {
    setInterval(() => {
      if (this.metricsBuffer.length > 0) {
        this.flushMetrics();
      }
    }, this.options.flushInterval);
  }

  private async flushMetrics() {
    if (this.options.metricsEndpoint) {
      try {
        await fetch(this.options.metricsEndpoint, {
          method: 'POST',
          headers: { 'Content-Type': 'application/json' },
          body: JSON.stringify(this.metricsBuffer)
        });
      } catch (error) {
        console.error('Failed to flush metrics:', error);
      }
    }
    this.metricsBuffer = [];
  }

  apply(compiler: Compiler) {
    // 1. File Change Detection
    compiler.hooks.watchRun.tap('RspackDevFeedbackPlugin', (compiler) => {
      const changedFiles = compiler.modifiedFiles;
      if (!changedFiles) return;
      
      for (const file of changedFiles) {
        const normalizedPath = this.normalizePath(file);
        this.changeMap.set(normalizedPath, {
          file: normalizedPath,
          changeDetectedAt: Date.now()
        });
      }
    });

    // 2. Compilation Start
    compiler.hooks.compile.tap('RspackDevFeedbackPlugin', () => {
      for (const [file, timing] of this.changeMap.entries()) {
        timing.compileStartAt = Date.now();
      }
    });

    // 3. Compilation Complete
    compiler.hooks.done.tap('RspackDevFeedbackPlugin', (stats) => {
      const now = Date.now();
      for (const [file, timing] of this.changeMap.entries()) {
        timing.compileDoneAt = now;
      }
    });

    // 4. Client Code Injection
    compiler.hooks.compilation.tap('RspackDevFeedbackPlugin', (compilation) => {
      // Use processAssets hook instead of afterHash
      compilation.hooks.processAssets.tap(
        {
          name: 'RspackDevFeedbackPlugin',
          stage: compilation.constructor.PROCESS_ASSETS_STAGE_ADDITIONS
        },
        () => {
          this.injectClientCode(compilation);
        }
      );
    });
  }

  private injectClientCode(compilation: Compilation) {
    const clientCode = this.generateClientCode();
    
    // Create a virtual module for the client code
    compilation.emitAsset(
      'rspack-dev-feedback-client.js',
      {
        source: () => clientCode,
        size: () => clientCode.length
      }
    );
  }

  private generateClientCode(): string {
    // Get WebSocket port dynamically
    const wsPort = (this.wsServer.address() as any).port;
    
    return `
      // Rspack Dev Feedback Client
      (() => {
        const ws = new WebSocket('ws://localhost:${wsPort}');
        let moduleUpdateTimer;

        // Handle different HMR implementations
        if (module.hot) {
          // Traditional Webpack-style HMR
          module.hot.addStatusHandler((status) => {
            handleHMRStatus(status);
          });
        } else if (window.__RSPACK_HMR__) {
          // Rspack-specific HMR implementation
          window.__RSPACK_HMR__.on('status', (status) => {
            handleHMRStatus(status);
          });
        }

        function handleHMRStatus(status) {
          if (status === 'idle' || status === 'ready') {
            clearTimeout(moduleUpdateTimer);
            moduleUpdateTimer = setTimeout(() => {
              ws.send(JSON.stringify({
                type: 'hmrApplied',
                timestamp: Date.now()
              }));
            }, 50); // Small delay to ensure DOM updates
          }
        }

        // Track actual DOM updates
        const observer = new MutationObserver(() => {
          if (moduleUpdateTimer) {
            clearTimeout(moduleUpdateTimer);
            moduleUpdateTimer = setTimeout(() => {
              ws.send(JSON.stringify({
                type: 'domUpdated',
                timestamp: Date.now()
              }));
            }, 50);
          }
        });

        observer.observe(document.body, {
          childList: true,
          subtree: true,
          attributes: true,
          characterData: true
        });
      })();
    `;
  }

  private normalizePath(filePath: string): string {
    const normalized = path.normalize(filePath);
    if (this.options.anonymizePaths) {
      // Replace absolute path with project-relative path
      return path.relative(process.cwd(), normalized);
    }
    return normalized;
  }

  private collectMetadata(file: string) {
    return {
      fileType: path.extname(file),
      fileSize: 0, // TODO: Implement file size calculation
      projectType: this.detectProjectType()
    };
  }

  private detectProjectType(): string {
    // TODO: Implement project type detection (React, Vue, etc.)
    return 'unknown';
  }
}

/**
 * Test harness for the plugin
 */
async function createTestHarness() {
  const webpack = require('@rspack/core');
  const MemoryFS = require('memory-fs');
  const plugin = new RspackDevFeedbackPlugin();

  const compiler = webpack({
    // Test configuration
    entry: './test/fixtures/entry.js',
    plugins: [plugin]
  });

  const fs = new MemoryFS();
  compiler.outputFileSystem = fs;

  return {
    compiler,
    plugin,
    async simulateFileChange(file: string) {
      // Simulate file change and measure timing
      const startTime = Date.now();
      await new Promise(resolve => compiler.hooks.done.tap('test', resolve));
      return Date.now() - startTime;
    }
  };
}

export { RspackDevFeedbackPlugin, createTestHarness };

Let me explain the key improvements and changes made based on the feedback:

1. Hook Selection & Compatibility:

• Switched from invalid to watchRun hook for more reliable file change detection
• Using processAssets instead of afterHash for client code injection
• Added support for both Webpack-style and Rspack-specific HMR implementations

2. Data Collection & Transmission:

• Implemented a dedicated WebSocket server for real-time metrics collection
• Added buffering and batch transmission of metrics
• Included mutation observer for accurate DOM update timing

3. Improved Timing Accuracy:

• Added multiple timing points: file change, compile start, compile done, HMR apply, and DOM updates
• Included small delays to ensure accurate timing of actual updates
• Added metadata collection for better analysis

4. Path Handling & Security:

• Added path normalization
• Included option for path anonymization
• Project-relative paths by default

5. Testing & Verification:

• Added test harness with MemoryFS
• Simulation capabilities for file changes
• Isolated testing environment

Key architectural decisions:

1. Using WebSocket instead of HTTP endpoints for real-time data collection
2. Supporting both traditional and Rspack-specific HMR implementations
3. Including DOM observation for more accurate update timing
4. Buffering metrics with configurable flush intervals

[joeldickson]

About using runtimeModule instead of processAssets. This would be much more efficient. Let me modify the implementation to use runtime modules.

/**
 * Implementation Plan for Rspack Developer Feedback Plugin
 * 
 * A plugin to measure and collect real-time developer feedback metrics in Rspack,
 * specifically tracking time-to-feedback for file changes during development.
 */

import { Compiler, Compilation } from '@rspack/core';
import WebSocket from 'ws';
import path from 'path';
import { createServer } from 'http';

interface TimingEntry {
  file: string;
  changeDetectedAt: number;
  compileStartAt?: number;
  compileDoneAt?: number;
  hmrAppliedAt?: number;
}

interface MetricsPayload {
  file: string;
  totalDuration: number;
  phases: {
    changeToCompile: number;
    compileDuration: number;
    hmrApplyDuration: number;
  };
  metadata: {
    fileType: string;
    fileSize: number;
    projectType: string;
  };
}

/**
 * Main plugin class for collecting development feedback metrics
 */
class RspackDevFeedbackPlugin {
  private changeMap = new Map<string, TimingEntry>();
  private wsServer: WebSocket.Server;
  private connections: Set<WebSocket> = new Set();
  private metricsBuffer: MetricsPayload[] = [];
  private options: {
    metricsEndpoint?: string;
    logLocally?: boolean;
    anonymizePaths?: boolean;
    flushInterval?: number;
  };

  constructor(options = {}) {
    this.options = {
      logLocally: true,
      anonymizePaths: true,
      flushInterval: 5000,
      ...options
    };
    
    // Initialize WebSocket server for real-time metrics
    const httpServer = createServer();
    this.wsServer = new WebSocket.Server({ server: httpServer });
    httpServer.listen(0, () => {
      const port = (httpServer.address() as any).port;
      console.log(`Metrics WebSocket server listening on port ${port}`);
    });

    this.setupWebSocket();
    this.startMetricsFlush();
  }

  private setupWebSocket() {
    this.wsServer.on('connection', (ws) => {
      this.connections.add(ws);
      ws.on('close', () => this.connections.delete(ws));
    });
  }

  private startMetricsFlush() {
    setInterval(() => {
      if (this.metricsBuffer.length > 0) {
        this.flushMetrics();
      }
    }, this.options.flushInterval);
  }

  private async flushMetrics() {
    if (this.options.metricsEndpoint) {
      try {
        await fetch(this.options.metricsEndpoint, {
          method: 'POST',
          headers: { 'Content-Type': 'application/json' },
          body: JSON.stringify(this.metricsBuffer)
        });
      } catch (error) {
        console.error('Failed to flush metrics:', error);
      }
    }
    this.metricsBuffer = [];
  }

  apply(compiler: Compiler) {
    // 1. File Change Detection
    compiler.hooks.watchRun.tap('RspackDevFeedbackPlugin', (compiler) => {
      const changedFiles = compiler.modifiedFiles;
      if (!changedFiles) return;
      
      for (const file of changedFiles) {
        const normalizedPath = this.normalizePath(file);
        this.changeMap.set(normalizedPath, {
          file: normalizedPath,
          changeDetectedAt: Date.now()
        });
      }
    });

    // 2. Compilation Start
    compiler.hooks.compile.tap('RspackDevFeedbackPlugin', () => {
      for (const [file, timing] of this.changeMap.entries()) {
        timing.compileStartAt = Date.now();
      }
    });

    // 3. Compilation Complete
    compiler.hooks.done.tap('RspackDevFeedbackPlugin', (stats) => {
      const now = Date.now();
      for (const [file, timing] of this.changeMap.entries()) {
        timing.compileDoneAt = now;
      }
    });

    // 4. Client Code Injection via Runtime Module
    compiler.hooks.compilation.tap('RspackDevFeedbackPlugin', (compilation) => {
      compilation.hooks.runtimeModule.tap('RspackDevFeedbackPlugin', () => {
        class DevFeedbackRuntimeModule extends compilation.runtimeModule {
          constructor() {
            super('rspack-dev-feedback');
          }

          generate() {
            return this.generateClientCode();
          }
        }

        compilation.addRuntimeModule(new DevFeedbackRuntimeModule());
      });
    });
  }

  private injectClientCode(compilation: Compilation) {
    const clientCode = this.generateClientCode();
    
    // Create a virtual module for the client code
    compilation.emitAsset(
      'rspack-dev-feedback-client.js',
      {
        source: () => clientCode,
        size: () => clientCode.length
      }
    );
  }

  private generateClientCode(): string {
    // Get WebSocket port dynamically
    const wsPort = (this.wsServer.address() as any).port;
    
    return `
      // Rspack Dev Feedback Client
      (() => {
        const ws = new WebSocket('ws://localhost:${wsPort}');
        let moduleUpdateTimer;

        // Handle different HMR implementations
        if (module.hot) {
          // Traditional Webpack-style HMR
          module.hot.addStatusHandler((status) => {
            handleHMRStatus(status);
          });
        } else if (window.__RSPACK_HMR__) {
          // Rspack-specific HMR implementation
          window.__RSPACK_HMR__.on('status', (status) => {
            handleHMRStatus(status);
          });
        }

        function handleHMRStatus(status) {
          if (status === 'idle' || status === 'ready') {
            clearTimeout(moduleUpdateTimer);
            moduleUpdateTimer = setTimeout(() => {
              ws.send(JSON.stringify({
                type: 'hmrApplied',
                timestamp: Date.now()
              }));
            }, 50); // Small delay to ensure DOM updates
          }
        }

        // Track actual DOM updates
        const observer = new MutationObserver(() => {
          if (moduleUpdateTimer) {
            clearTimeout(moduleUpdateTimer);
            moduleUpdateTimer = setTimeout(() => {
              ws.send(JSON.stringify({
                type: 'domUpdated',
                timestamp: Date.now()
              }));
            }, 50);
          }
        });

        observer.observe(document.body, {
          childList: true,
          subtree: true,
          attributes: true,
          characterData: true
        });
      })();
    `;
  }

  private normalizePath(filePath: string): string {
    const normalized = path.normalize(filePath);
    if (this.options.anonymizePaths) {
      // Replace absolute path with project-relative path
      return path.relative(process.cwd(), normalized);
    }
    return normalized;
  }

  private collectMetadata(file: string) {
    return {
      fileType: path.extname(file),
      fileSize: 0, // TODO: Implement file size calculation
      projectType: this.detectProjectType()
    };
  }

  private detectProjectType(): string {
    // TODO: Implement project type detection (React, Vue, etc.)
    return 'unknown';
  }
}

/**
 * Test harness for the plugin
 */
async function createTestHarness() {
  const webpack = require('@rspack/core');
  const MemoryFS = require('memory-fs');
  const plugin = new RspackDevFeedbackPlugin();

  const compiler = webpack({
    // Test configuration
    entry: './test/fixtures/entry.js',
    plugins: [plugin]
  });

  const fs = new MemoryFS();
  compiler.outputFileSystem = fs;

  return {
    compiler,
    plugin,
    async simulateFileChange(file: string) {
      // Simulate file change and measure timing
      const startTime = Date.now();
      await new Promise(resolve => compiler.hooks.done.tap('test', resolve));
      return Date.now() - startTime;
    }
  };
}

export { RspackDevFeedbackPlugin, createTestHarness };

This optimization brings several benefits:

1. Performance:

   • Runtime modules are much more efficient than processing assets
   • They're designed specifically for injecting runtime code
   • Avoids unnecessary asset processing overhead

2. Better Integration:

   • Runtime modules are properly integrated into Rspack's module system
   • They get proper source maps and minification
   • They're handled at the right phase of compilation

3. Vite Compatibility:

   • For Vite, we can use its virtual module system similarly:

   // Vite version
   const virtualModuleId = 'virtual:dev-feedback'
   const resolvedVirtualModuleId = '\0' + virtualModuleId
   
   export default function devFeedbackPlugin() {
     return {
       name: 'dev-feedback',
       resolveId(id) {
         if (id === virtualModuleId) {
           return resolvedVirtualModuleId
         }
       },
       load(id) {
         if (id === resolvedVirtualModuleId) {
           return generateClientCode()
         }
       }
     }
   }