From 1426aa3478285fbed7b2de09aa2eb9a267b83da6 Mon Sep 17 00:00:00 2001 From: Arkadiusz Bulski Date: Thu, 5 Apr 2018 15:15:49 +0200 Subject: [PATCH 1/6] xml comments overall --- KaitaiStream.cs | 237 ++++++++++++++++++++++++++---------------------- 1 file changed, 127 insertions(+), 110 deletions(-) diff --git a/KaitaiStream.cs b/KaitaiStream.cs index ea2b48f..c91df7e 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -7,34 +7,50 @@ namespace Kaitai { /// - /// The base Kaitai steam which exposes an API for the Kaitai Struct framework. - /// It's based off a BinaryReader, which is a little-endian reader. + /// The base Kaitai stream which exposes an API for the Kaitai Struct framework. + /// It's based off a BinaryReader, which is a little-endian reader. /// public partial class KaitaiStream : BinaryReader { #region Constructors + /// + /// Create a KaitaiStream backed by abstract stream. It could be in-memory buffer or open file. + /// public KaitaiStream(Stream stream) : base(stream) { } - /// - /// Creates a KaitaiStream backed by a file (RO) - /// - public KaitaiStream(string file) : base(File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read)) + /// + /// Create a KaitaiStream by opening a file in read-only binary mode (FileStream). + /// + public KaitaiStream(string filename) : base(File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { } - /// - ///Creates a KaitaiStream backed by a byte buffer - /// - public KaitaiStream(byte[] bytes) : base(new MemoryStream(bytes)) + /// + /// Create a KaitaiStream backed by in-memory buffer. + /// + public KaitaiStream(byte[] data) : base(new MemoryStream(data)) { } + /// + /// Temporary 64-bit buffer for leftover bits left from an unaligned bit + /// read operation. Following unaligned bit operations would consume bits + /// left in this buffer first, and then, if needed, would continue + /// consuming bytes from the stream. + /// private ulong Bits = 0; + /// + /// Number of bits left in Bits buffer. + /// private int BitsLeft = 0; + /// + /// Caches the current platform endianness and allows emitted bytecode to be optimized. Thanks to @Arlorean. + /// https://github.com/kaitai-io/kaitai_struct_csharp_runtime/pull/9 + /// static readonly bool IsLittleEndian = BitConverter.IsLittleEndian; #endregion @@ -42,7 +58,8 @@ public KaitaiStream(byte[] bytes) : base(new MemoryStream(bytes)) #region Stream positioning /// - /// Check if the stream position is at the end of the stream + /// Check if the stream position is at the end of the stream (at EOF). + /// WARNING: This requires a stream that supports seeking (memory-based or file-based). /// public bool IsEof { @@ -50,16 +67,18 @@ public bool IsEof } /// - /// Seek to a specific position from the beginning of the stream + /// Move the stream to a specified absolute position. + /// WARNING: This requires a stream that supports seeking (memory-based or file-based). /// - /// The position to seek to + /// The position to seek to, as non-negative integer public void Seek(long position) { BaseStream.Seek(position, SeekOrigin.Begin); } /// - /// Get the current position in the stream + /// Get the current position within the stream. + /// WARNING: This requires a stream that supports seeking (memory-based or file-based). /// public long Pos { @@ -67,7 +86,8 @@ public long Pos } /// - /// Get the total length of the stream (ie. file size) + /// Get the total length of the stream (ie. file size). + /// WARNING: This requires a stream that supports seeking (memory-based or file-based). /// public long Size { @@ -81,9 +101,8 @@ public long Size #region Signed /// - /// Read a signed byte from the stream + /// Read a signed byte (1 byte) from the stream. /// - /// public sbyte ReadS1() { return ReadSByte(); @@ -92,27 +111,24 @@ public sbyte ReadS1() #region Big-endian /// - /// Read a signed short from the stream (big endian) + /// Read a signed short (2 bytes) from the stream (big endian). /// - /// public short ReadS2be() { return BitConverter.ToInt16(ReadBytesNormalisedBigEndian(2), 0); } /// - /// Read a signed int from the stream (big endian) + /// Read a signed int (4 bytes) from the stream (big endian). /// - /// public int ReadS4be() { return BitConverter.ToInt32(ReadBytesNormalisedBigEndian(4), 0); } /// - /// Read a signed long from the stream (big endian) + /// Read a signed long (8 bytes) from the stream (big endian). /// - /// public long ReadS8be() { return BitConverter.ToInt64(ReadBytesNormalisedBigEndian(8), 0); @@ -123,27 +139,24 @@ public long ReadS8be() #region Little-endian /// - /// Read a signed short from the stream (little endian) + /// Read a signed short (2 bytes) from the stream (little endian). /// - /// public short ReadS2le() { return BitConverter.ToInt16(ReadBytesNormalisedLittleEndian(2), 0); } /// - /// Read a signed int from the stream (little endian) + /// Read a signed int (4 bytes) from the stream (little endian). /// - /// public int ReadS4le() { return BitConverter.ToInt32(ReadBytesNormalisedLittleEndian(4), 0); } /// - /// Read a signed long from the stream (little endian) + /// Read a signed long (8 bytes) from the stream (little endian). /// - /// public long ReadS8le() { return BitConverter.ToInt64(ReadBytesNormalisedLittleEndian(8), 0); @@ -156,9 +169,8 @@ public long ReadS8le() #region Unsigned /// - /// Read an unsigned byte from the stream + /// Read an unsigned byte (1 bytes) from the stream. /// - /// public byte ReadU1() { return ReadByte(); @@ -167,27 +179,24 @@ public byte ReadU1() #region Big-endian /// - /// Read an unsigned short from the stream (big endian) + /// Read an unsigned short (2 bytes) from the stream (big endian). /// - /// public ushort ReadU2be() { return BitConverter.ToUInt16(ReadBytesNormalisedBigEndian(2), 0); } /// - /// Read an unsigned int from the stream (big endian) + /// Read an unsigned int (4 bytes) from the stream (big endian). /// - /// public uint ReadU4be() { return BitConverter.ToUInt32(ReadBytesNormalisedBigEndian(4), 0); } /// - /// Read an unsigned long from the stream (big endian) + /// Read an unsigned long (8 bytes) from the stream (big endian). /// - /// public ulong ReadU8be() { return BitConverter.ToUInt64(ReadBytesNormalisedBigEndian(8), 0); @@ -198,27 +207,24 @@ public ulong ReadU8be() #region Little-endian /// - /// Read an unsigned short from the stream (little endian) + /// Read an unsigned short (2 bytes) from the stream (little endian). /// - /// public ushort ReadU2le() { return BitConverter.ToUInt16(ReadBytesNormalisedLittleEndian(2), 0); } /// - /// Read an unsigned int from the stream (little endian) + /// Read an unsigned int (4 bytes) from the stream (little endian). /// - /// public uint ReadU4le() { return BitConverter.ToUInt32(ReadBytesNormalisedLittleEndian(4), 0); } /// - /// Read an unsigned long from the stream (little endian) + /// Read an unsigned long (8 bytes) from the stream (little endian). /// - /// public ulong ReadU8le() { return BitConverter.ToUInt64(ReadBytesNormalisedLittleEndian(8), 0); @@ -235,18 +241,16 @@ public ulong ReadU8le() #region Big-endian /// - /// Read a single-precision floating point value from the stream (big endian) + /// Read a single-precision floating point value (4 bytes) from the stream (big endian). /// - /// public float ReadF4be() { return BitConverter.ToSingle(ReadBytesNormalisedBigEndian(4), 0); } /// - /// Read a double-precision floating point value from the stream (big endian) + /// Read a double-precision floating point value (8 bytes) from the stream (big endian). /// - /// public double ReadF8be() { return BitConverter.ToDouble(ReadBytesNormalisedBigEndian(8), 0); @@ -257,18 +261,16 @@ public double ReadF8be() #region Little-endian /// - /// Read a single-precision floating point value from the stream (little endian) + /// Read a single-precision floating point value (4 bytes) from the stream (little endian). /// - /// public float ReadF4le() { return BitConverter.ToSingle(ReadBytesNormalisedLittleEndian(4), 0); } /// - /// Read a double-precision floating point value from the stream (little endian) + /// Read a double-precision floating point value (8 bytes) from the stream (little endian). /// - /// public double ReadF8le() { return BitConverter.ToDouble(ReadBytesNormalisedLittleEndian(8), 0); @@ -280,12 +282,29 @@ public double ReadF8le() #region Unaligned bit values + /// + /// Clears the temporary buffer which holds not-yet-consumed parts of + /// the byte, which might have left over after last unaligned bit read + /// operation. Effectively, aligns the pointer in the stream to next + /// whole byte, discarding the rest of partially byte, if it existed. + /// Does nothing if unaligned bit reading is not used. + /// public void AlignToByte() { Bits = 0; BitsLeft = 0; } + /// + /// Read next n bits from the stream. By convention, starts from most + /// significant bits first and goes to least significant bits. + /// + /// + /// If n is not a multiple of 8, then bits left over from partially + /// consumed byte would be stored in stream internal buffer. Subsequent + /// calls to this method would consume bits from that buffer first, and + /// then proceed with fetching the new bytes from the stream. + /// public ulong ReadBitsInt(int n) { int bitsNeeded = n - BitsLeft; @@ -329,25 +348,23 @@ private static ulong GetMaskOnes(int n) #region Byte arrays /// - /// Read a fixed number of bytes from the stream + /// Read a fixed number of bytes from the stream. /// - /// The number of bytes to read - /// + /// Amount of bytes to read, as non-negative integer public byte[] ReadBytes(long count) { if (count < 0 || count > Int32.MaxValue) throw new ArgumentOutOfRangeException("requested " + count + " bytes, while only non-negative int32 amount of bytes possible"); byte[] bytes = base.ReadBytes((int) count); if (bytes.Length < count) - throw new EndOfStreamException("requested " + count + " bytes, but got only " + bytes.Length + " bytes"); + throw new EndOfStreamException("requested " + count + " bytes, but stream returned only " + bytes.Length + " bytes"); return bytes; } /// - /// Read a fixed number of bytes from the stream + /// Read a fixed number of bytes from the stream. /// - /// The number of bytes to read - /// + /// Amount of bytes to read, as non-negative integer public byte[] ReadBytes(ulong count) { if (count > Int32.MaxValue) @@ -359,10 +376,9 @@ public byte[] ReadBytes(ulong count) } /// - /// Read bytes from the stream in little endian format and convert them to the endianness of the current platform + /// Read bytes from the stream in little endian format and convert them to the endianness of the current platform. /// - /// The number of bytes to read - /// An array of bytes that matches the endianness of the current platform + /// Amount of bytes to read, as non-negative integer protected byte[] ReadBytesNormalisedLittleEndian(int count) { byte[] bytes = ReadBytes(count); @@ -371,10 +387,9 @@ protected byte[] ReadBytesNormalisedLittleEndian(int count) } /// - /// Read bytes from the stream in big endian format and convert them to the endianness of the current platform + /// Read bytes from the stream in big endian format and convert them to the endianness of the current platform. /// - /// The number of bytes to read - /// An array of bytes that matches the endianness of the current platform + /// Amount of bytes to read, as non-negative integer protected byte[] ReadBytesNormalisedBigEndian(int count) { byte[] bytes = ReadBytes(count); @@ -383,22 +398,21 @@ protected byte[] ReadBytesNormalisedBigEndian(int count) } /// - /// Read all the remaining bytes from the stream until the end is reached + /// Read all the remaining bytes from the stream until EOF is reached. + /// WARNING: This requires a stream that supports seeking (memory-based or file-based). /// - /// public byte[] ReadBytesFull() { return ReadBytes(BaseStream.Length - BaseStream.Position); } /// - /// Read a terminated string from the stream + /// Read bytes from the stream, until either terminating byte or EOF is encountered. /// - /// The string terminator value - /// True to include the terminator in the returned string - /// True to consume the terminator byte before returning - /// True to throw an error when the EOS was reached before the terminator - /// + /// The terminating byte, as integer + /// True to include the terminator in the returned bytes + /// True to consume the terminator before returning bytes + /// True to throw an error when the EOF was reached before the terminator public byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consumeTerminator, bool eosError) { List bytes = new List(); @@ -406,7 +420,7 @@ public byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consum { if (IsEof) { - if (eosError) throw new EndOfStreamException(string.Format("End of stream reached, but no terminator `{0}` found", terminator)); + if (eosError) throw new EndOfStreamException(string.Format("End of stream reached, but no terminator {0} found", terminator)); break; } @@ -423,10 +437,9 @@ public byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consum } /// - /// Read a specific set of bytes and assert that they are the same as an expected result + /// Read a matching amount of bytes and assert that it matches the expected data. Returns same data, or throws an exception. /// - /// The expected result - /// + /// The expected data, as byte array public byte[] EnsureFixedContents(byte[] expected) { byte[] bytes = ReadBytes(expected.Length); @@ -446,6 +459,11 @@ public byte[] EnsureFixedContents(byte[] expected) return bytes; } + /// + /// Perform right-to-left strip on a byte array. + /// + /// The data, as byte array + /// The padding byte, as integer public static byte[] BytesStripRight(byte[] src, byte padByte) { int newLen = src.Length; @@ -457,6 +475,12 @@ public static byte[] BytesStripRight(byte[] src, byte padByte) return dst; } + /// + /// Perform left-to-right search of specified terminating byte, and cutoff remaining bytes. + /// + /// The data, as byte array + /// The terminating byte, as integer + /// True to include the terminating byte in result public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTerminator) { int newLen = 0; @@ -478,11 +502,10 @@ public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTer #region Byte array processing /// - /// Performs XOR processing with given data, XORing every byte of the input with a single value. + /// Perform XOR processing between given data and single-byte key. /// - /// The data toe process - /// The key value to XOR with - /// Processed data + /// The data to process, as byte array + /// The key to XOR with, as integer public byte[] ProcessXor(byte[] value, int key) { byte[] result = new byte[value.Length]; @@ -494,12 +517,10 @@ public byte[] ProcessXor(byte[] value, int key) } /// - /// Performs XOR processing with given data, XORing every byte of the input with a key - /// array, repeating from the beginning of the key array if necessary + /// Perform XOR processing between given data and multiple-byte key. /// - /// The data toe process - /// The key array to XOR with - /// Processed data + /// The data to process, as byte array + /// The key to XOR with, as byte array public byte[] ProcessXor(byte[] value, byte[] key) { int keyLen = key.Length; @@ -512,13 +533,12 @@ public byte[] ProcessXor(byte[] value, byte[] key) } /// - /// Performs a circular left rotation shift for a given buffer by a given amount of bits. + /// Perform circular left rotation shift for a given data by a given amount of bits. /// Pass a negative amount to rotate right. /// - /// The data to rotate - /// The number of bytes to rotate by - /// - /// + /// The data to rotate, as byte array + /// The amount to rotate by (in bits), as integer + /// The size of group in which rotation happens, as non-negative integer public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) { if (amount > 7 || amount < -7) throw new ArgumentException("Rotation of more than 7 cannot be performed.", "amount"); @@ -542,10 +562,9 @@ public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) } /// - /// Inflates a deflated zlib byte stream + /// Inflate a previously deflated zlib byte stream. /// - /// The data to deflate - /// The deflated result + /// The data to deflate, as byte array public byte[] ProcessZlib(byte[] data) { // See RFC 1950 (https://tools.ietf.org/html/rfc1950) @@ -581,16 +600,15 @@ public byte[] ProcessZlib(byte[] data) #region Misc utility methods /// - /// Performs modulo operation between two integers. + /// Perform modulo operation between two integers. + /// NOTE: Result is always non-negative and within `0..b-1`. /// /// - /// This method is required because C# lacks a "true" modulo - /// operator, the % operator rather being the "remainder" - /// operator. We want mod operations to always be positive. + /// This method is required because C# lacks a "true" modulo operator, + /// the % operator rather being the "division remainder" operator. /// - /// The value to be divided - /// The value to divide by. Must be greater than zero. - /// The result of the modulo opertion. Will always be positive. + /// The value to be divided, as integer + /// The value to divide by, as positive integer public static int Mod(int a, int b) { if (b <= 0) throw new ArgumentException("Divisor of mod operation must be greater than zero.", "b"); @@ -600,16 +618,15 @@ public static int Mod(int a, int b) } /// - /// Performs modulo operation between two integers. + /// Perform modulo operation between two integers. + /// NOTE: Result is always non-negative and within `0..b-1`. /// /// - /// This method is required because C# lacks a "true" modulo - /// operator, the % operator rather being the "remainder" - /// operator. We want mod operations to always be positive. + /// This method is required because C# lacks a "true" modulo operator, + /// the % operator rather being the "division remainder" operator. /// - /// The value to be divided - /// The value to divide by. Must be greater than zero. - /// The result of the modulo opertion. Will always be positive. + /// The value to be divided, as integer + /// The value to divide by, as positive integer public static long Mod(long a, long b) { if (b <= 0) throw new ArgumentException("Divisor of mod operation must be greater than zero.", "b"); @@ -619,11 +636,11 @@ public static long Mod(long a, long b) } /// - /// Compares two byte arrays in lexicographical order. + /// Compare two byte arrays in lexicographical order. The arrays may not be equal length. /// - /// negative number if a is less than b, 0 if a is equal to b, positive number if a is greater than b. + /// Negative number if a is less than b, 0 if a is equal to b, positive number if a is greater than b. /// First byte array to compare - /// Second byte array to compare. + /// Second byte array to compare public static int ByteArrayCompare(byte[] a, byte[] b) { if (a == b) From cb66e6c50a3bd4d2423e8fe957650f63834e054f Mon Sep 17 00:00:00 2001 From: Arkadiusz Bulski Date: Thu, 12 Apr 2018 00:46:25 +0200 Subject: [PATCH 2/6] EnsureFixedContents: optimised --- KaitaiStream.cs | 54 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/KaitaiStream.cs b/KaitaiStream.cs index c91df7e..d0cbca0 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -444,16 +444,9 @@ public byte[] EnsureFixedContents(byte[] expected) { byte[] bytes = ReadBytes(expected.Length); - if (bytes.Length != expected.Length) + if (ByteArrayEqual(bytes, expected) == false) { - throw new Exception(string.Format("Expected bytes: {0} ({1} bytes), Instead got: {2} ({3} bytes)", Convert.ToBase64String(expected), expected.Length, Convert.ToBase64String(bytes), bytes.Length)); - } - for (int i = 0; i < bytes.Length; i++) - { - if (bytes[i] != expected[i]) - { - throw new Exception(string.Format("Expected bytes: {0} ({1} bytes), Instead got: {2} ({3} bytes)", Convert.ToBase64String(expected), expected.Length, Convert.ToBase64String(bytes), bytes.Length)); - } + throw new Exception(string.Format("Expected: {0} , Actual: {1}", Convert.ToBase64String(expected), Convert.ToBase64String(bytes) )); } return bytes; @@ -648,18 +641,49 @@ public static int ByteArrayCompare(byte[] a, byte[] b) int al = a.Length; int bl = b.Length; int minLen = al < bl ? al : bl; - for (int i = 0; i < minLen; i++) { + + for (int i = 0; i < minLen; i++) + { int cmp = a[i] - b[i]; if (cmp != 0) return cmp; } // Reached the end of at least one of the arrays - if (al == bl) { - return 0; - } else { - return al - bl; - } + return al == bl ? 0 : al - bl; + } + + /// + /// Compare two byte arrays for exact equality. + /// + /// + /// .NET has Array.Equals and == operators, but those check object identity. + /// + public static bool ByteArrayEqual(byte[] a, byte[] b) + { + if (a == b) + return true; + + if (a.Length != b.Length) + return false; + + for (int i=0; i + /// Check if byte array is all zeroes. + /// + public static bool IsByteArrayZero(byte[] a) + { + for (int i=0; i Date: Fri, 6 Apr 2018 22:28:22 +0200 Subject: [PATCH 3/6] replaced Seek(Pos-1) with Seek(-1, CUR) --- KaitaiStream.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/KaitaiStream.cs b/KaitaiStream.cs index d0cbca0..ec6eb46 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -427,8 +427,10 @@ public byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consum byte b = ReadByte(); if (b == terminator) { - if (includeTerminator) bytes.Add(b); - if (!consumeTerminator) Seek(Pos - 1); + if (includeTerminator) + bytes.Add(b); + if (!consumeTerminator) + BaseStream.Seek(-1, SeekOrigin.Current); break; } bytes.Add(b); From da5a05075291db9c01c1bf31c2ef754c51230580 Mon Sep 17 00:00:00 2001 From: Arkadiusz Bulski Date: Sat, 7 Apr 2018 01:31:00 +0200 Subject: [PATCH 4/6] BytesStripRight and BytesTerminate: can return original array --- KaitaiStream.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/KaitaiStream.cs b/KaitaiStream.cs index ec6eb46..b0c3a07 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -456,15 +456,21 @@ public byte[] EnsureFixedContents(byte[] expected) /// /// Perform right-to-left strip on a byte array. + /// WARNING: Can return original byte array. /// /// The data, as byte array /// The padding byte, as integer public static byte[] BytesStripRight(byte[] src, byte padByte) { int newLen = src.Length; + int maxLen = src.Length; + while (newLen > 0 && src[newLen - 1] == padByte) newLen--; + if (newLen == maxLen) + return src; + byte[] dst = new byte[newLen]; Array.Copy(src, dst, newLen); return dst; @@ -472,6 +478,7 @@ public static byte[] BytesStripRight(byte[] src, byte padByte) /// /// Perform left-to-right search of specified terminating byte, and cutoff remaining bytes. + /// WARNING: Can return original byte array. /// /// The data, as byte array /// The terminating byte, as integer @@ -487,6 +494,9 @@ public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTer if (includeTerminator && newLen < maxLen) newLen++; + if (newLen == maxLen) + return src; + byte[] dst = new byte[newLen]; Array.Copy(src, dst, newLen); return dst; From cd2ee76bfb5c3b0e556a6aaa56ecc9063c9276fb Mon Sep 17 00:00:00 2001 From: Arkadiusz Bulski Date: Thu, 12 Apr 2018 00:56:48 +0200 Subject: [PATCH 5/6] ProcessXor: optimisations --- KaitaiStream.cs | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/KaitaiStream.cs b/KaitaiStream.cs index b0c3a07..7bd69d1 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -508,32 +508,50 @@ public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTer /// /// Perform XOR processing between given data and single-byte key. + /// WARNING: May return same byte array if key is zero. /// - /// The data to process, as byte array + /// The data to process, as byte array /// The key to XOR with, as integer - public byte[] ProcessXor(byte[] value, int key) + public byte[] ProcessXor(byte[] data, int key) { - byte[] result = new byte[value.Length]; - for (int i = 0; i < value.Length; i++) + if (key == 0) + return data; + + byte[] result = new byte[data.Length]; + + for (int i = 0; i < data.Length; i++) { - result[i] = (byte)(value[i] ^ key); + result[i] = (byte)(data[i] ^ key); } + return result; } /// /// Perform XOR processing between given data and multiple-byte key. + /// WARNING: May return same byte array if key is zero. /// - /// The data to process, as byte array + /// The data to process, as byte array /// The key to XOR with, as byte array - public byte[] ProcessXor(byte[] value, byte[] key) + public byte[] ProcessXor(byte[] data, byte[] key) { + if (key.Length == 1) + return ProcessXor(data, key[0]); + if (key.Length <= 64 && IsByteArrayZero(key)) + return data; + + byte[] result = new byte[data.Length]; int keyLen = key.Length; - byte[] result = new byte[value.Length]; - for (int i = 0, j = 0; i < value.Length; i++, j = (j + 1) % keyLen) + + int k = 0; + for (int i = 0; i < data.Length; i++) { - result[i] = (byte)(value[i] ^ key[j]); + result[i] = (byte)(data[i] ^ key[k]); + k++; + if (k == keyLen) + k = 0; } + return result; } From a531feed969fe45ea80e364ab7d592a671d46e3a Mon Sep 17 00:00:00 2001 From: Arkadiusz Bulski Date: Thu, 12 Apr 2018 00:57:29 +0200 Subject: [PATCH 6/6] ProcessRotateLeft: optimisations --- KaitaiStream.cs | 100 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 15 deletions(-) diff --git a/KaitaiStream.cs b/KaitaiStream.cs index 7bd69d1..a5f9a00 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -53,6 +53,11 @@ public KaitaiStream(byte[] data) : base(new MemoryStream(data)) /// static readonly bool IsLittleEndian = BitConverter.IsLittleEndian; + static KaitaiStream() + { + computeSingleRotations(); + } + #endregion #region Stream positioning @@ -555,33 +560,98 @@ public byte[] ProcessXor(byte[] data, byte[] key) return result; } + private static byte[][] precomputedSingleRotations; + + /// + /// Speeds up ProcessRotateLeft by precomputing translations tables. + /// In effect only when groupSize is 1. + /// + private static void computeSingleRotations() + { + precomputedSingleRotations = new byte[8][]; + for (int amount = 1; amount < 8; amount++) + { + int anti_amount = 8 - amount; + byte[] translate = new byte[256]; + for (int i = 0; i < 256; i++) + { + // formula taken from: http://stackoverflow.com/a/812039 + translate[i] = (byte) ((i << amount) | (i >> anti_amount)); + } + precomputedSingleRotations[amount] = translate; + } + } + /// /// Perform circular left rotation shift for a given data by a given amount of bits. /// Pass a negative amount to rotate right. + /// WARNING: May return same byte array if amount is zero (modulo-wise). /// /// The data to rotate, as byte array - /// The amount to rotate by (in bits), as integer - /// The size of group in which rotation happens, as non-negative integer + /// The amount to rotate by (in bits), as integer, negative for right rotation + /// The size of group in which rotation happens, as positive integer public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) { - if (amount > 7 || amount < -7) throw new ArgumentException("Rotation of more than 7 cannot be performed.", "amount"); - if (amount < 0) amount += 8; // Rotation of -2 is the same as rotation of +6 + if (groupSize < 1) + throw new ArgumentException("group size must be at least 1 to be valid", "groupSize"); + + amount = Mod(amount, groupSize * 8); + if (amount == 0) + return data; - byte[] r = new byte[data.Length]; - switch (groupSize) + int amount_bytes = amount / 8; + byte[] result = new byte[data.Length]; + + if (groupSize == 1) { - case 1: - for (int i = 0; i < data.Length; i++) + byte[] translate = precomputedSingleRotations[amount]; + + for (int i = 0; i < data.Length; i++) + { + result[i] = translate[data[i]]; + } + return result; + } + + if (data.Length % groupSize != 0) + throw new Exception("data length must be a multiple of group size"); + + if (amount % 8 == 0) + { + int[] indices = new int[groupSize]; + for (int i = 0; i < groupSize; i++) + { + indices[i] = (i + amount_bytes) % groupSize; + } + for (int i = 0; i < data.Length; i += groupSize) + { + for (int k = 0; k < groupSize; k++) { - byte bits = data[i]; - // http://stackoverflow.com/a/812039 - r[i] = (byte) ((bits << amount) | (bits >> (8 - amount))); + result[i+k] = data[i + indices[k]]; } - break; - default: - throw new NotImplementedException(string.Format("Unable to rotate a group of {0} bytes yet", groupSize)); + } + return result; + } + + { + int amount1 = amount % 8; + int amount2 = 8 - amount1; + int[] indices1 = new int[groupSize]; + int[] indices2 = new int[groupSize]; + for (int i = 0; i < groupSize; i++) + { + indices1[i] = (i + amount_bytes) % groupSize; + indices2[i] = (i + 1 + amount_bytes) % groupSize; + } + for (int i = 0; i < data.Length; i += groupSize) + { + for (int k = 0; k < groupSize; k++) + { + result[i+k] = (byte) ((data[i + indices1[k]] << amount1) | (data[i + indices2[k]] >> amount2)); + } + } + return result; } - return r; } ///