From 1e8ac3e3aaa6a97d447f5d89c522d5662363e6dd Mon Sep 17 00:00:00 2001 From: Yury Delendik Date: Mon, 14 Sep 2015 13:04:03 -0500 Subject: [PATCH 1/4] Adds ability to feed swfm data in chunks. --- src/base/dataBuffer.ts | 11 +++++ src/gfx/test/playbackEaselHost.ts | 26 ++++++++++-- src/gfx/test/recorder.ts | 67 ++++++++++++++++++++++++++++--- 3 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src/base/dataBuffer.ts b/src/base/dataBuffer.ts index 5500c66269..bec27baee6 100644 --- a/src/base/dataBuffer.ts +++ b/src/base/dataBuffer.ts @@ -160,6 +160,17 @@ module Shumway.ArrayUtilities { return clone; } + /** + * Moves rest of the data starting from current position to the beginning of + * the buffer. The length will be reduced by the position, and the position + * will be set to 0. + */ + compact(): void { + this._u8.set(this._u8.subarray(this._position, this._length), 0); + this._length -= this._position; + this._position = 0; + } + /** * By default, we only have a byte view. All other views are |null|. */ diff --git a/src/gfx/test/playbackEaselHost.ts b/src/gfx/test/playbackEaselHost.ts index a9bbff7595..97f29d423d 100644 --- a/src/gfx/test/playbackEaselHost.ts +++ b/src/gfx/test/playbackEaselHost.ts @@ -28,6 +28,7 @@ module Shumway.GFX.Test { export class PlaybackEaselHost extends EaselHost { private _parser: MovieRecordParser; private _lastTimestamp: number; + private _parsingInProgress: boolean; public ignoreTimestamps: boolean = false; public alwaysRenderFrame: boolean = false; @@ -38,6 +39,9 @@ module Shumway.GFX.Test { public constructor(easel: Easel) { super(easel); + this._parsingInProgress = false; + this._parser = new MovieRecordParser(); + this._lastTimestamp = 0; } public get cpuTime(): number { @@ -49,15 +53,22 @@ module Shumway.GFX.Test { xhr.open('GET', url, true); xhr.responseType = 'arraybuffer'; xhr.onload = function () { - this.playBytes(new Uint8Array(xhr.response)); + var data = new Uint8Array(xhr.response); + this.playBytes(data); + this.finish(); }.bind(this); xhr.send(); } private playBytes(data: Uint8Array) { - this._parser = new MovieRecordParser(data); - this._lastTimestamp = 0; - this._parseNext(); + this._parser.push(data); + if (!this._parsingInProgress) { + this._parseNext(); + } + } + + private finish() { + this._parser.close(); } onSendUpdates(updates: DataBuffer, assets: Array) { @@ -74,7 +85,13 @@ module Shumway.GFX.Test { private _parseNext() { var type = this._parser.readNextRecord(); + if (type === MovieRecordType.Incomplete) { + this._parsingInProgress = false; + return; + } + if (type !== MovieRecordType.None) { + this._parsingInProgress = true; var runRecordBound = this._runRecord.bind(this); var interval = this._parser.currentTimestamp - this._lastTimestamp; this._lastTimestamp = this._parser.currentTimestamp; @@ -87,6 +104,7 @@ module Shumway.GFX.Test { setTimeout(runRecordBound, interval); } } else { + this._parsingInProgress = false; if (this.onComplete) { this.onComplete(); } diff --git a/src/gfx/test/recorder.ts b/src/gfx/test/recorder.ts index 6ea0f53db2..852ef528bd 100644 --- a/src/gfx/test/recorder.ts +++ b/src/gfx/test/recorder.ts @@ -118,6 +118,7 @@ module Shumway.GFX.Test { } export const enum MovieRecordType { + Incomplete = -1, None = 0, PlayerCommand = 1, PlayerCommandAsync = 2, @@ -150,7 +151,8 @@ module Shumway.GFX.Test { } public dump() { - var parser = new MovieRecordParser(this._recording.getBytes()); + var parser = new MovieRecordParser(); + parser.push(this._recording.getBytes()); parser.dump(); } @@ -283,26 +285,77 @@ module Shumway.GFX.Test { } } + const enum MovieCompressionType { + None, + ZLib, + Lzma + } + + const enum MovieRecordParserState { + Initial, + Parsing, + Ended + } + + var MovieHeaderSize = 4; + var MovieRecordHeaderSize = 12; + export class MovieRecordParser { private _buffer: DataBuffer; + private _state: MovieRecordParserState; + private _comressionType: MovieCompressionType; + private _closed: boolean; public currentTimestamp: number; public currentType: MovieRecordType; public currentData: DataBuffer; - constructor(data: Uint8Array) { + constructor () { + this._state = MovieRecordParserState.Initial; this._buffer = new DataBuffer(); + this._closed = false; + } + + public push(data: Uint8Array){ + this._buffer.compact(); + + var savedPosition = this._buffer.position; + this._buffer.position = this._buffer.length; this._buffer.writeRawBytes(data); - this._buffer.position = 4; + this._buffer.position = savedPosition; + } + + public close() { + this._closed = true; } public readNextRecord(): MovieRecordType { + if (this._state === MovieRecordParserState.Initial) { + if (this._buffer.position + MovieHeaderSize > this._buffer.length) { + return MovieRecordType.Incomplete; + } + this._buffer.position += MovieHeaderSize; + this._comressionType = MovieCompressionType.None; + this._state = MovieRecordParserState.Parsing; + } + if (this._buffer.position >= this._buffer.length) { - return MovieRecordType.None; + return this._closed ? MovieRecordType.None : MovieRecordType.Incomplete; + } + + if (this._buffer.position + MovieRecordHeaderSize > this._buffer.length) { + return MovieRecordType.Incomplete; } + var timestamp: number = this._buffer.readInt(); var type: MovieRecordType = this._buffer.readInt(); - var length: number = this._buffer.readInt(); + var length: number = this._buffer.readInt() >>> 0; + + if (this._buffer.position + MovieRecordHeaderSize + length > this._buffer.length) { + this._buffer.position -= MovieRecordHeaderSize; + return MovieRecordType.Incomplete; + } + var data: DataBuffer = null; if (length > 0) { @@ -350,7 +403,7 @@ module Shumway.GFX.Test { public dump() { var type: MovieRecordType; - while ((type = this.readNextRecord())) { + while ((type = this.readNextRecord()) !== MovieRecordType.None) { console.log('record ' + type + ' @' + this.currentTimestamp); switch (type) { case MovieRecordType.PlayerCommand: @@ -366,6 +419,8 @@ module Shumway.GFX.Test { case MovieRecordType.Image: console.log(this.parseImage()); break; + case MovieRecordType.Incomplete: + return; } } } From 5d580648f9a470d3c01ee6f2e48d0941a5811c94 Mon Sep 17 00:00:00 2001 From: Yury Delendik Date: Mon, 14 Sep 2015 15:24:32 -0500 Subject: [PATCH 2/4] Changes DataBuffer.readRawBytes signature. --- src/base/dataBuffer.ts | 15 +++++++++++++-- src/flash/display/BitmapData.ts | 3 ++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/base/dataBuffer.ts b/src/base/dataBuffer.ts index bec27baee6..94822347c2 100644 --- a/src/base/dataBuffer.ts +++ b/src/base/dataBuffer.ts @@ -626,8 +626,19 @@ module Shumway.ArrayUtilities { } } - readRawBytes(): Int8Array { - return new Int8Array(this._buffer, 0, this._length); + readRawBytes(length: number): Uint8Array { + if (length < 0) { + release || assert((this).sec); + (this).sec.throwError('RangeError', Errors.ParamRangeError); + } + var position = this._position; + if (position + length > this._length) { + release || assert((this).sec); + (this).sec.throwError('flash.errors.EOFError', Errors.EOFError); + } + var result = new Uint8Array(this._u8.subarray(position, position + length)); + this._position = position + 4; + return result; } writeUTF(value: string): void { diff --git a/src/flash/display/BitmapData.ts b/src/flash/display/BitmapData.ts index 243a386344..65b76456bf 100644 --- a/src/flash/display/BitmapData.ts +++ b/src/flash/display/BitmapData.ts @@ -794,7 +794,8 @@ module Shumway.AVMX.AS.flash.display { } setPixels(rect: flash.geom.Rectangle, inputByteArray: flash.utils.ByteArray): void { - this._putPixelData(rect, new Int32Array(inputByteArray.readRawBytes())); + var data = inputByteArray.getBytes(); + this._putPixelData(rect, new Int32Array(data.buffer, 0, data.length >> 2)); } setVector(rect: flash.geom.Rectangle, inputVector: Uint32Vector): void { From f7e1feafc9fd0cb559bf0979c22c68975b205f1a Mon Sep 17 00:00:00 2001 From: Yury Delendik Date: Wed, 14 Oct 2015 19:50:35 -0500 Subject: [PATCH 3/4] Creation and playback of compressed SWFM files. --- src/base/dataBuffer.ts | 8 +- src/gfx/test/recorder.ts | 74 ++++++++++++++++-- utils/compressswfm.js | 162 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+), 10 deletions(-) create mode 100644 utils/compressswfm.js diff --git a/src/base/dataBuffer.ts b/src/base/dataBuffer.ts index 94822347c2..24e34dbb31 100644 --- a/src/base/dataBuffer.ts +++ b/src/base/dataBuffer.ts @@ -166,8 +166,12 @@ module Shumway.ArrayUtilities { * will be set to 0. */ compact(): void { - this._u8.set(this._u8.subarray(this._position, this._length), 0); - this._length -= this._position; + var position = this._position; + if (position === 0) { + return; // nothing to compact + } + this._u8.set(this._u8.subarray(position, this._length), 0); + this._length -= position; this._position = 0; } diff --git a/src/gfx/test/recorder.ts b/src/gfx/test/recorder.ts index 852ef528bd..229eb52896 100644 --- a/src/gfx/test/recorder.ts +++ b/src/gfx/test/recorder.ts @@ -17,6 +17,9 @@ module Shumway.GFX.Test { import DataBuffer = Shumway.ArrayUtilities.DataBuffer; import PlainObjectDataBuffer = Shumway.ArrayUtilities.PlainObjectDataBuffer; + import IDataDecoder = Shumway.ArrayUtilities.IDataDecoder; + import Inflate = Shumway.ArrayUtilities.Inflate; + import LzmaDecoder = Shumway.ArrayUtilities.LzmaDecoder; enum MovieRecordObjectType { Undefined = 0, @@ -300,11 +303,19 @@ module Shumway.GFX.Test { var MovieHeaderSize = 4; var MovieRecordHeaderSize = 12; + class IdentityDataTransform implements IDataDecoder { + onData: (data: Uint8Array) => void = null; + onError: (e) => void = null; + push(data: Uint8Array) { this.onData(data); } + close() {} + } + export class MovieRecordParser { private _buffer: DataBuffer; private _state: MovieRecordParserState; private _comressionType: MovieCompressionType; private _closed: boolean; + private _transform: IDataDecoder; public currentTimestamp: number; public currentType: MovieRecordType; @@ -316,7 +327,46 @@ module Shumway.GFX.Test { this._closed = false; } - public push(data: Uint8Array){ + public push(data: Uint8Array): void { + if (this._state === MovieRecordParserState.Initial) { + // SWFM file starts from 4 bytes header: "MSWF", "MSWC", or "MSWZ" + var needToRead = MovieHeaderSize - this._buffer.length; + this._buffer.writeRawBytes(data.subarray(0, needToRead)); + if (MovieHeaderSize > this._buffer.length) { + return; + } + this._buffer.position = 0; + var headerBytes = this._buffer.readRawBytes(MovieHeaderSize); + if (headerBytes[0] !== 0x4D || headerBytes[1] !== 0x53 || headerBytes[2] !== 0x57 || + (headerBytes[3] !== 0x46 && headerBytes[3] !== 0x43 && headerBytes[3] !== 0x5A)) { + console.warn('Invalid SWFM header, stopping parsing'); + return; + } + switch (headerBytes[3]) { + case 0x46: // 'F' + this._comressionType = MovieCompressionType.None; + this._transform = new IdentityDataTransform(); + break; + case 0x43: // 'C' + this._comressionType = MovieCompressionType.ZLib; + this._transform = Inflate.create(true); + break; + case 0x5A: // 'Z' + this._comressionType = MovieCompressionType.Lzma; + this._transform = new LzmaDecoder(false); + break; + } + + this._transform.onData = this._pushTransformed.bind(this); + this._transform.onError = this._errorTransformed.bind(this); + this._state = MovieRecordParserState.Parsing; + this._buffer.clear(); + data = data.subarray(needToRead); + } + this._transform.push(data); + } + + private _pushTransformed(data: Uint8Array): void { this._buffer.compact(); var savedPosition = this._buffer.position; @@ -326,21 +376,29 @@ module Shumway.GFX.Test { } public close() { + this._transform.close(); this._closed = true; } + private _errorTransformed() { + console.warn('Error in SWFM stream'); + this.close(); + } + public readNextRecord(): MovieRecordType { if (this._state === MovieRecordParserState.Initial) { - if (this._buffer.position + MovieHeaderSize > this._buffer.length) { - return MovieRecordType.Incomplete; - } - this._buffer.position += MovieHeaderSize; - this._comressionType = MovieCompressionType.None; - this._state = MovieRecordParserState.Parsing; + return MovieRecordType.Incomplete; + } + if (this._state === MovieRecordParserState.Ended) { + return MovieRecordType.None; } if (this._buffer.position >= this._buffer.length) { - return this._closed ? MovieRecordType.None : MovieRecordType.Incomplete; + if (this._closed) { + this._state = MovieRecordParserState.Ended; + return MovieRecordType.None; + } + return MovieRecordType.Incomplete; } if (this._buffer.position + MovieRecordHeaderSize > this._buffer.length) { diff --git a/utils/compressswfm.js b/utils/compressswfm.js new file mode 100644 index 0000000000..23102aeb24 --- /dev/null +++ b/utils/compressswfm.js @@ -0,0 +1,162 @@ +/* + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var fs = require('fs'); +var path = require('path'); +var temp = require('temp'); +var spawn = require('child_process').spawn; + +// simple args parsing +var compressMethod = null; +var outputPath = null; +var inputPath = null; +for (var i = 2; i < process.argv.length;) { + var cmd = process.argv[i++]; + switch (cmd) { + case '--lzma': + case '-z': + compressMethod = 'lzma'; + break; + case '--zlib': + case '-c': + compressMethod = 'zlib'; + break; + case '--out': + case '-o': + outputPath = process.argv[i++]; + break; + default: + inputPath = cmd; // .swfm is expected + break; + } +} + +function createRawSWFM(callback) { + // Create a temp file without 'MSWF' header. + fs.readFile(inputPath, function (err, data) { + if (err) return callback(err); + var header = data.slice(0, 4).toString(); + if (header !== 'MSWF') { + return callback(new Error('swfm file header is not found: found \"' + header + '\"')); + } + temp.open('swfm', function (err, info) { + if (err) return callback(err); + fs.write(info.fd, data, 4, data.length - 4, function (err) { + if (err) return callback(err); + fs.close(info.fd, function (err) { + if (err) return callback(err); + callback(null, info.path); + }); + }); + }); + }); +} + +function compressViaGZip(rawDataPath, compressedDataPath, callback) { + // Running gzip. + var proc = spawn('gzip', ['-k9n', rawDataPath]); + var resultPath = rawDataPath + '.gz'; + proc.on('close', function (code) { + if (code !== 0 || !fs.existsSync(resultPath)) { + callback(new Error('Unable to run gzip')); + return; + } + // Prepending MSWC and zlib header before compressed data. + fs.writeFile(compressedDataPath, new Buffer("MSWC\u0078\u00DA", 'ascii'), function (err) { + if (err) return callback(err); + fs.readFile(resultPath, function (err, data) { + if (data[0] !== 0x1F || data[1] !== 0x8B || data[2] !== 0x08 || data[3] !== 0) { + return callback(new Error('Invalid gzip result')); + } + var dataStart = 10; + var dataEnd = data.length - 8; + + // Appending adler32 at the end of data. + var a = 1, b = 0; + for (var i = dataStart; i < dataEnd; ++i) { + a = (a + (data[i] & 0xff)) % 65521; + b = (b + a) % 65521; + } + var adler32 = (b << 16) | a; + data[dataEnd] = (adler32 >> 24) & 255; + data[dataEnd + 1] = (adler32 >> 16) & 255; + data[dataEnd + 2] = (adler32 >> 8) & 255; + data[dataEnd + 3] = adler32 & 255; + + + if (err) return callback(err); + fs.appendFile(compressedDataPath, data.slice(dataStart, dataEnd + 4), function (err) { + fs.unlink(resultPath, callback); + }); + }); + }); + }); +} + +function compressViaLzma(rawDataPath, compressedDataPath, callback) { + // Running lzma. + var proc = spawn('lzma', ['-zk9e', rawDataPath]); + var resultPath = rawDataPath + '.lzma'; + proc.on('close', function (code) { + if (code !== 0 || !fs.existsSync(resultPath)) { + callback(new Error('Unable to run lzma')); + return; + } + // Prepending MSWZ before lzma compressed data. + fs.writeFile(compressedDataPath, "MSWZ", function (err) { + if (err) return callback(err); + fs.readFile(resultPath, function (err, data) { + if (err) return callback(err); + fs.appendFile(compressedDataPath, data, function (err) { + fs.unlink(resultPath, callback); + }); + }); + }); + }); +} + +function printUsage() { + console.info('Usage: node compressswfm.js [-c|-z] inputFile [-o outputFile]'); +} + +if (!inputPath || !compressMethod) { + printUsage(); + process.exit(1); +} + +if (!outputPath) { + var ext = path.extname(inputPath); + outputPath = inputPath.slice(0, -ext.length) + '.' + compressMethod + ext; + console.log('Compressed results at ' + outputPath); +} + +createRawSWFM(function (err, path) { + if (err) throw err; + switch (compressMethod) { + case 'zlib': + compressViaGZip(path, outputPath, function (err) { + if (err) throw err; + console.log('Done. Compressed using ZLib.'); + }); + break; + case 'lzma': + compressViaLzma(path, outputPath, function (err) { + if (err) throw err; + console.log('Done. Compressed using LZMA.'); + }); + break; + } +}); From 6afd7d1eeca9c32c7c12f54a5ad4f25ca11e9a8b Mon Sep 17 00:00:00 2001 From: Yury Delendik Date: Wed, 14 Oct 2015 19:58:15 -0500 Subject: [PATCH 4/4] Renames dataBuffer compact() to the removeHead(). --- src/base/dataBuffer.ts | 8 ++++---- src/gfx/test/recorder.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/base/dataBuffer.ts b/src/base/dataBuffer.ts index 24e34dbb31..323515dfbb 100644 --- a/src/base/dataBuffer.ts +++ b/src/base/dataBuffer.ts @@ -161,11 +161,11 @@ module Shumway.ArrayUtilities { } /** - * Moves rest of the data starting from current position to the beginning of - * the buffer. The length will be reduced by the position, and the position - * will be set to 0. + * Removes bytes from start to the current position, and moves the rest of the data + * to the beginning of the buffer. The length will be reduced by the number of + * the removed bytes, and the position will be set to 0. */ - compact(): void { + removeHead(): void { var position = this._position; if (position === 0) { return; // nothing to compact diff --git a/src/gfx/test/recorder.ts b/src/gfx/test/recorder.ts index 229eb52896..2e126100a4 100644 --- a/src/gfx/test/recorder.ts +++ b/src/gfx/test/recorder.ts @@ -367,7 +367,7 @@ module Shumway.GFX.Test { } private _pushTransformed(data: Uint8Array): void { - this._buffer.compact(); + this._buffer.removeHead(); var savedPosition = this._buffer.position; this._buffer.position = this._buffer.length;