feat(streams): capture MP4 stream

Adds a `capture` method to the pipelines that produce MP4 data. It stores all data in a buffer at start of movie, and passes it to the provided callback when capture ends. The latter can be triggered by calling the returned trigger function, or automatically when the buffer is full. Co-authored-by: Rikard Tegnander <[email protected]> Co-authored-by: Victor Ingvarsson <[email protected]>
AxisCommunications · Jan 11, 2025 · 0f40d48 · 0f40d48
1 parent 8114f37
commit 0f40d48
Show file tree

Hide file tree

Showing 10 changed files with 244 additions and 207 deletions.
diff --git a/example-streams-web/camera/simple-mp4-player.js b/example-streams-web/camera/simple-mp4-player.js
@@ -18,7 +18,6 @@ const authorize = async (host) => {
   }
 }
 
-let pipeline
 const play = (host) => {
   // Grab a reference to the video element
   const mediaElement = document.querySelector('video')
@@ -31,8 +30,10 @@ const play = (host) => {
   pipeline.start().catch((err) => {
     console.error(err)
   })
+  return pipeline
 }
 
+let pipeline
 // Each time a device ip is entered, authorize and then play
 const playButton = document.querySelector('#play')
 playButton.addEventListener('click', async () => {
@@ -45,3 +46,26 @@ playButton.addEventListener('click', async () => {
 
   pipeline = play(host)
 })
+
+let stopCapture
+const startCaptureButton = document.querySelector('#startCapture')
+startCaptureButton.addEventListener('click', async () => {
+  if (!pipeline) {
+    console.error('No pipeline')
+    return
+  }
+
+  stopCapture = await pipeline.capture((bytes) => {
+    console.log('Capture finished!', bytes.byteLength)
+    console.log(bytes)
+  })
+})
+
+const stopCaptureButton = document.querySelector('#stopCapture')
+stopCaptureButton.addEventListener('click', async () => {
+  if (!stopCapture) {
+    console.error('Capture not started!')
+    return
+  }
+  stopCapture()
+})
diff --git a/example-streams-web/camera/simple-mp4.html b/example-streams-web/camera/simple-mp4.html
@@ -1,26 +1,25 @@
 <!DOCTYPE html>
 <html>
-  <head>
-    <title>Example streaming MP4 video from Axis camera</title>
-  </head>
 
-  <body>
+<head>
+  <title>Example streaming MP4 video from Axis camera</title>
+</head>
+
+<body>
+  <div>
     <div>
-      <div>
-        <label for="device">Device IP:</label>
-        <input id="device" type="text" placeholder="192.168.0.90" />
-      </div>
-      <button id="play">Play</button>
-    </div>
-    <div style="position: fixed; width: 1280px; height: 720px">
-      <video
-        style="position: absolute; width: 100%; height: 100%"
-        autoplay
-        muted
-        controls
-      ></video>
+      <label for="device">Device IP:</label>
+      <input id="device" type="text" placeholder="192.168.0.90" />
     </div>
-    <script src="/media-stream-library.min.js"></script>
-    <script src="simple-mp4-player.js"></script>
-  </body>
+    <button id="play">Play</button>
+    <button id="startCapture">Start capture</button>
+    <button id="stopCapture">Stop capture</button>
+  </div>
+  <div>
+    <video style="width: 100%; max-width: 1280px" autoplay muted controls></video>
+  </div>
+  <script src="/media-stream-library.min.js"></script>
+  <script src="simple-mp4-player.js"></script>
+</body>
+
 </html>
diff --git a/example-streams-web/favicon.ico b/example-streams-web/favicon.ico
diff --git a/streams/src/components/index.ts b/streams/src/components/index.ts
@@ -4,7 +4,6 @@ export * from './utils'
 
 export * from './adapter'
 export * from './canvas'
-export * from './mp4-capture'
 export * from './mp4-muxer'
 export * from './mse-sink'
 export * from './rtp'

diff --git a/streams/src/components/mp4-capture.ts b/streams/src/components/mp4-capture.ts
diff --git a/streams/src/components/mse-sink.ts b/streams/src/components/mse-sink.ts
@@ -15,7 +15,10 @@ import { IsomMessage } from './types'
  */
 export class MseSink {
   public readonly mediaSource: MediaSource = new MediaSource()
-  public writable: WritableStream<IsomMessage>
+  public readonly writable: WritableStream<IsomMessage>
+
+  /** A function that will peek at ISOM messages, useful for example for capturing the MP4 data. */
+  public onMessage?: (msg: IsomMessage) => void
 
   private lastCheckpointTime: number
   private sourceBuffer?: Promise<SourceBuffer>
@@ -57,6 +60,10 @@ export class MseSink {
           await freeBuffer(sourceBuffer, checkpoint)
         }
 
+        if (this.onMessage) {
+          this.onMessage(msg)
+        }
+
         await appendBuffer(sourceBuffer, msg.data)
       },
       close: async () => {

diff --git a/streams/src/http-mp4-pipeline.ts b/streams/src/http-mp4-pipeline.ts
@@ -1,4 +1,6 @@
 import { Adapter, IsomMessage, MseSink } from './components'
+import { logDebug } from './log'
+import { setupMp4Capture } from './mp4-capture'
 
 export interface HttpMp4Config {
   uri: string
@@ -18,43 +20,69 @@ export interface HttpMp4Config {
 export class HttpMp4Pipeline {
   public onHeaders?: (headers: Headers) => void
   public onServerClose?: () => void
-  /** Initiates the stream and resolves when the media stream has completed */
-  public start: () => Promise<void>
 
-  private readonly mediaElement: HTMLVideoElement
-  private readonly abortController: AbortController
+  private abortController?: AbortController
   private downloadedBytes: number = 0
+  private options?: RequestInit
+  private readonly mediaElement: HTMLVideoElement
+  private uri: string
 
   constructor(config: HttpMp4Config) {
     const { uri, options, mediaElement } = config
 
+    this.uri = uri
+    this.options = options
+
     this.mediaElement = mediaElement
+  }
+
+  /** Initiates the stream and resolves when the media stream has completed. */
+  public async start(msgHandler?: (msg: IsomMessage) => void) {
+    this.abortController?.abort('stream restarted')
+
     this.abortController = new AbortController()
 
-    this.start = () =>
-      fetch(uri, { signal: this.abortController.signal, ...options })
-        .then(({ headers, body }) => {
-          const mimeType = headers.get('Content-Type')
-          if (!mimeType) {
-            throw new Error('missing MIME type in HTTP response headers')
-          }
-          if (body === null) {
-            throw new Error('missing body in HTTP response')
-          }
-          const adapter = new Adapter<IsomMessage>((chunk) => {
-            this.downloadedBytes += chunk.byteLength
-            return new IsomMessage({ data: chunk })
-          })
-          const mseSink = new MseSink(mediaElement, mimeType)
-          return body.pipeThrough(adapter).pipeTo(mseSink.writable)
-        })
-        .catch((err) => {
-          console.error('failed to stream media:', err)
-        })
+    const { ok, status, statusText, headers, body } = await fetch(this.uri, {
+      signal: this.abortController.signal,
+      ...this.options,
+    })
+
+    if (!ok) {
+      throw new Error(`response not ok, status: ${statusText} (${status})`)
+    }
+
+    const mimeType = headers.get('Content-Type')
+    if (!mimeType) {
+      throw new Error('missing MIME type in HTTP response headers')
+    }
+
+    if (body === null) {
+      throw new Error('missing body in HTTP response')
+    }
+
+    const adapter = new Adapter<IsomMessage>((chunk) => {
+      this.downloadedBytes += chunk.byteLength
+      return new IsomMessage({ data: chunk })
+    })
+
+    const mseSink = new MseSink(this.mediaElement, mimeType)
+    if (msgHandler) {
+      mseSink.onMessage = msgHandler
+    }
+
+    body
+      .pipeThrough(adapter)
+      .pipeTo(mseSink.writable)
+      .then(() => {
+        logDebug(`http-mp4 pipeline ended: stream ended`)
+      })
+      .catch((err) => {
+        logDebug(`http-mp4 pipeline ended: ${err}`)
+      })
   }
 
   public close() {
-    this.abortController.abort()
+    this.abortController?.abort('Closed by user')
   }
 
   public get currentTime() {
@@ -72,4 +100,14 @@ export class HttpMp4Pipeline {
   public get byteLength() {
     return this.downloadedBytes
   }
+
+  /** Refresh the stream and passes the captured MP4 data to the provided
+   * callback. Capture can be ended by calling the returned trigger, or
+   * if the buffer reaches max size. */
+  public async capture(callback: (bytes: Uint8Array) => void) {
+    this.close()
+    const { capture, triggerEnd } = setupMp4Capture(callback)
+    await this.start(capture)
+    return triggerEnd
+  }
 }
diff --git a/streams/src/mp4-capture.ts b/streams/src/mp4-capture.ts
@@ -0,0 +1,69 @@
+import { logInfo as logDebug } from './log'
+
+import { IsomMessage } from './components/types'
+import { encode } from './components/utils/bytes'
+
+const MAX_BUFFER = 225000000 // 5 min at a rate of 6 Mbit/s
+
+// Detect the start of the movie by detecting an ftyp box.
+const magicHeader = encode('ftyp')
+function isFtypIsom(box: Uint8Array): boolean {
+  const header = box.subarray(4, 8)
+  return magicHeader.every((byte, i) => byte === header[i])
+}
+
+/** Given a callback and max buffer size, returns two functions, one that takes
+ * MP4 data (as ISOM message) and stores that data whenever it detects the start
+ * of a movie, and a function that triggers the end of data storage. The trigger
+ * is called automatically if the buffer is full. */
+export function setupMp4Capture(
+  cb: (bytes: Uint8Array) => void,
+  bufferSize = MAX_BUFFER
+): {
+  capture: (msg: IsomMessage) => void
+  triggerEnd: () => void
+} {
+  let active = true
+  let buffer = new Uint8Array(bufferSize)
+  let bufferOffset = 0
+  let startOfMovie = false
+
+  const triggerEnd = () => {
+    active = false
+    logDebug(`stop MP4 capture, collected ${bufferOffset} bytes`)
+    try {
+      cb(buffer.subarray(0, bufferOffset))
+    } catch (err) {
+      console.error('capture callback failed:', err)
+    }
+  }
+
+  const capture = (msg: IsomMessage) => {
+    if (!active) {
+      return
+    }
+
+    // Arrival of ISOM with MIME type indicates new movie, start recording if active.
+    if (!startOfMovie) {
+      if (isFtypIsom(msg.data)) {
+        startOfMovie = true
+        logDebug('detected start of movie, proceeding with MP4 capture')
+      } else {
+        return
+      }
+    }
+
+    // If movie started, record all ISOM (MP4) boxes
+    if (bufferOffset < buffer.byteLength - msg.data.byteLength) {
+      buffer.set(msg.data, bufferOffset)
+      bufferOffset += msg.data.byteLength
+    } else {
+      triggerEnd()
+    }
+  }
+
+  return {
+    capture,
+    triggerEnd,
+  }
+}