diff --git a/Directory.Build.props b/Directory.Build.props index 9f4d998c..717e1398 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -7,6 +7,7 @@ $(ProjectBaseVersion) $(ProjectVersion) $(ProjectVersion) + latest diff --git a/ValveKeyValue/ValveKeyValue.Test/ApiSurfaceTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/ApiSurfaceTestCase.cs index 0fa2f964..0b2ef39c 100644 --- a/ValveKeyValue/ValveKeyValue.Test/ApiSurfaceTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/ApiSurfaceTestCase.cs @@ -83,6 +83,37 @@ static void GenerateTypeApiSurface(StringBuilder sb, Type type) sb.Append('\n'); } + var constructors = type + .GetConstructors(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .Where(t => !t.IsPrivate && !t.IsAssembly && !t.IsFamilyAndAssembly) + .OrderBy(t => t.Name, StringComparer.InvariantCulture) + .ThenBy(t => string.Join(", ", t.GetParameters().Select(GetParameterAsString)), StringComparer.InvariantCulture); + + foreach (var constructor in constructors) + { + sb.Append(" "); + + if (constructor.IsPublic) + { + sb.Append("public"); + } + else + { + sb.Append("protected"); + } + + if (constructor.IsStatic) + { + sb.Append(" static"); + } + + sb.Append(' '); + sb.Append(constructor.Name); + sb.Append('('); + sb.Append(string.Join(", ", constructor.GetParameters().Select(GetParameterAsString))); + sb.Append(");\n"); + } + var methods = type .GetMethods(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) .Where(t => !t.IsPrivate && !t.IsAssembly && !t.IsFamilyAndAssembly) diff --git a/ValveKeyValue/ValveKeyValue.Test/Binary/SimpleBinaryTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/Binary/SimpleBinaryTestCase.cs index 81bcd01d..24a2f958 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Binary/SimpleBinaryTestCase.cs +++ b/ValveKeyValue/ValveKeyValue.Test/Binary/SimpleBinaryTestCase.cs @@ -56,7 +56,7 @@ public void SetUp() 0x06, // pointer: ptr = 0x11223344 0x70, 0x74, 0x72, 0x00, 0x44, 0x33, 0x22, 0x11, - 0x07, // uint64: long = 0x1122334455667788 + 0x07, // uint64: lng = 0x1122334455667788 0x6C, 0x6E, 0x67, 0x00, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x0A, // int64, i64 = 0x0102030405070809 diff --git a/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableFromScratchTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableFromScratchTestCase.cs new file mode 100644 index 00000000..0f79bb23 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableFromScratchTestCase.cs @@ -0,0 +1,29 @@ +using System.Linq; + +namespace ValveKeyValue.Test +{ + class StringTableFromScratchTestCase + { + [Test] + public void PopulatesStringTableDuringSerialization() + { + var kv = new KVObject("root", + [ + new KVObject("key", "value"), + new KVObject("child", [ + new KVObject("key", 123), + ]), + ]); + + var stringTable = new StringTable(); + + var serializer = KVSerializer.Create(KVSerializationFormat.KeyValues1Binary); + + using var ms = new MemoryStream(); + serializer.Serialize(ms, kv, new KVSerializerOptions { StringTable = stringTable }); + + var strings = stringTable.ToArray(); + Assert.That(strings, Is.EqualTo(new[] { "root", "key", "child" })); + } + } +} diff --git a/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableTestCase.cs new file mode 100644 index 00000000..280c593b --- /dev/null +++ b/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableTestCase.cs @@ -0,0 +1,87 @@ +using System.Linq; + +namespace ValveKeyValue.Test +{ + class StringTableTestCase + { + [Test] + public void IsNotNull() + => Assert.That(obj, Is.Not.Null); + + [Test] + public void HasName() + => Assert.That(obj.Name, Is.EqualTo("TestObject")); + + [Test] + public void IsObjectWithChildren() + => Assert.That(obj.Value.ValueType, Is.EqualTo(KVValueType.Collection)); + + [TestCase(ExpectedResult = 5)] + public int HasChildren() + => obj.Children.Count(); + + [TestCase("key", "value", typeof(string))] + [TestCase("int", 0x01020304, typeof(int))] + [TestCase("flt", 1234.5678f, typeof(float))] + [TestCase("lng", 0x1122334455667788, typeof(ulong))] + [TestCase("i64", 0x0102030405060708, typeof(long))] + public void HasNamedChildWithValue(string name, object value, Type valueType) + { + Assert.That(Convert.ChangeType(obj[name], valueType), Is.EqualTo(value)); + } + + [Test] + public void SymmetricStringTableSerialization() + { + var serializer = KVSerializer.Create(KVSerializationFormat.KeyValues1Binary); + + using var ms = new MemoryStream(); + serializer.Serialize(ms, obj, new KVSerializerOptions { StringTable = new(TestStringTable) }); + + Assert.That(ms.ToArray(), Is.EqualTo(TestData.ToArray())); + } + + KVObject obj; + + [OneTimeSetUp] + public void SetUp() + { + obj = KVSerializer.Create(KVSerializationFormat.KeyValues1Binary) + .Deserialize( + TestData.ToArray(), + new KVSerializerOptions { StringTable = new(TestStringTable) }); + } + + static string[] TestStringTable => [ + "flt", + "i64", + "int", + "key", + "lng", + "TestObject" + ]; + + static ReadOnlySpan TestData => + [ + 0x00, // object: TestObject + 0x05, 0x00, 0x00, 0x00, // stringTable[5] = "TestObject", + 0x01, // string: key = value + 0x03, 0x00, 0x00, 0x00, // stringTable[3] = "key", + 0x76, 0x61, 0x6C, 0x75, 0x65, 0x00, + 0x02, // int32: int = 0x01020304 + 0x02, 0x00, 0x00, 0x00, // stringTable[2] = "int" + 0x04, 0x03, 0x02, 0x01, + 0x03, // float32: flt = 1234.5678f + 0x00, 0x00, 0x00, 0x00, // stringTable[0] = "flt" + 0x2B, 0x52, 0x9A, 0x44, + 0x07, // uint64: lng = 0x1122334455667788 + 0x04, 0x00, 0x00, 0x00, // stringTable[4] = "lng" + 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, + 0x0A, // int64, i64 = 0x0102030405070809 + 0x01, 0x00, 0x00, 0x00, // stringTable[1] = "i64" + 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, + 0x08, // end object + 0x08, // end document + ]; + } +} diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt b/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt index b05adec7..39620ce3 100644 --- a/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt +++ b/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt @@ -5,6 +5,9 @@ public interface ValveKeyValue.IIncludedFileLoader public class ValveKeyValue.KeyValueException { + public .ctor(); + public .ctor(string message); + public .ctor(string message, Exception inner); protected void add_SerializeObjectState(EventHandler`1[[System.Runtime.Serialization.SafeSerializationEventArgs]] value); public bool Equals(object obj); protected void Finalize(); @@ -31,6 +34,7 @@ public class ValveKeyValue.KeyValueException public class ValveKeyValue.KVArrayValue { + public .ctor(); public void Add(ValveKeyValue.KVValue value); public void AddRange(System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVValue]] values); public void Clear(); @@ -74,6 +78,8 @@ public class ValveKeyValue.KVArrayValue public class ValveKeyValue.KVBinaryBlob { + public .ctor(byte[] value); + public .ctor(Memory`1[[byte]] value); public bool Equals(object obj); protected void Finalize(); public Memory`1[[byte]] get_Bytes(); @@ -104,6 +110,7 @@ public class ValveKeyValue.KVBinaryBlob public class ValveKeyValue.KVDocument { + public .ctor(string name, ValveKeyValue.KVValue value); public void Add(ValveKeyValue.KVObject value); public bool Equals(object obj); protected void Finalize(); @@ -121,6 +128,7 @@ public class ValveKeyValue.KVDocument public sealed class ValveKeyValue.KVIgnoreAttribute { + public .ctor(); public bool Equals(object obj); protected void Finalize(); public object get_TypeId(); @@ -134,6 +142,8 @@ public sealed class ValveKeyValue.KVIgnoreAttribute public class ValveKeyValue.KVObject { + public .ctor(string name, System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVObject]] items); + public .ctor(string name, ValveKeyValue.KVValue value); public void Add(ValveKeyValue.KVObject value); public bool Equals(object obj); protected void Finalize(); @@ -151,6 +161,7 @@ public class ValveKeyValue.KVObject public sealed class ValveKeyValue.KVPropertyAttribute { + public .ctor(string propertyName); public bool Equals(object obj); protected void Finalize(); public string get_PropertyName(); @@ -200,6 +211,7 @@ public class ValveKeyValue.KVSerializer public sealed class ValveKeyValue.KVSerializerOptions { + public .ctor(); public bool Equals(object obj); protected void Finalize(); public System.Collections.Generic.IList`1[[string]] get_Conditions(); @@ -207,17 +219,20 @@ public sealed class ValveKeyValue.KVSerializerOptions public bool get_EnableValveNullByteBugBehavior(); public ValveKeyValue.IIncludedFileLoader get_FileLoader(); public bool get_HasEscapeSequences(); + public ValveKeyValue.StringTable get_StringTable(); public int GetHashCode(); public Type GetType(); protected object MemberwiseClone(); public void set_EnableValveNullByteBugBehavior(bool value); public void set_FileLoader(ValveKeyValue.IIncludedFileLoader value); public void set_HasEscapeSequences(bool value); + public void set_StringTable(ValveKeyValue.StringTable value); public string ToString(); } public class ValveKeyValue.KVValue { + protected .ctor(); public bool Equals(object obj); protected void Finalize(); public ValveKeyValue.KVValue get_Item(string key); @@ -295,3 +310,20 @@ public sealed enum ValveKeyValue.KVValueType public string ToString(string format, IFormatProvider provider); } +public sealed class ValveKeyValue.StringTable +{ + public .ctor(); + public .ctor(int capacity); + public .ctor(System.Collections.Generic.IList`1[[string]] values); + public void Add(string value); + public bool Equals(object obj); + protected void Finalize(); + public string get_Item(int index); + public int GetHashCode(); + public int GetOrAdd(string value); + public Type GetType(); + protected object MemberwiseClone(); + public string[] ToArray(); + public string ToString(); +} + diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs index 7c4be940..d987ca9d 100644 --- a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs +++ b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs @@ -8,7 +8,7 @@ class KV1BinaryReader : IVisitingReader { public const int BinaryMagicHeader = 0x564B4256; // VBKV - public KV1BinaryReader(Stream stream, IVisitationListener listener) + public KV1BinaryReader(Stream stream, IVisitationListener listener, StringTable stringTable) { Require.NotNull(stream, nameof(stream)); Require.NotNull(listener, nameof(listener)); @@ -20,12 +20,14 @@ public KV1BinaryReader(Stream stream, IVisitationListener listener) this.stream = stream; this.listener = listener; + this.stringTable = stringTable; reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true); } readonly Stream stream; readonly BinaryReader reader; readonly IVisitationListener listener; + readonly StringTable stringTable; bool disposed; KV1BinaryNodeType endMarker = KV1BinaryNodeType.End; @@ -74,9 +76,20 @@ void ReadObjectCore() } } + string ReadKeyForNextValue() + { + if (stringTable is not null) + { + var index = reader.ReadInt32(); + return stringTable[index]; + } + + return Encoding.UTF8.GetString(ReadNullTerminatedBytes()); + } + void ReadValue(KV1BinaryNodeType type) { - var name = Encoding.UTF8.GetString(ReadNullTerminatedBytes()); + var name = ReadKeyForNextValue(); KVValue value; switch (type) diff --git a/ValveKeyValue/ValveKeyValue/KVSerializer.cs b/ValveKeyValue/ValveKeyValue/KVSerializer.cs index 965765d8..35fac7fc 100644 --- a/ValveKeyValue/ValveKeyValue/KVSerializer.cs +++ b/ValveKeyValue/ValveKeyValue/KVSerializer.cs @@ -115,7 +115,7 @@ IVisitingReader MakeReader(Stream stream, IParsingVisitationListener listener, K return format switch { KVSerializationFormat.KeyValues1Text => new KV1TextReader(new StreamReader(stream), listener, options), - KVSerializationFormat.KeyValues1Binary => new KV1BinaryReader(stream, listener), + KVSerializationFormat.KeyValues1Binary => new KV1BinaryReader(stream, listener, options.StringTable), _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Invalid serialization format."), }; } @@ -128,9 +128,10 @@ IVisitationListener MakeSerializer(Stream stream, KVSerializerOptions options) return format switch { KVSerializationFormat.KeyValues1Text => new KV1TextSerializer(stream, options), - KVSerializationFormat.KeyValues1Binary => new KV1BinarySerializer(stream), + KVSerializationFormat.KeyValues1Binary => new KV1BinarySerializer(stream, options.StringTable), _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Invalid serialization format."), }; + ; } } } diff --git a/ValveKeyValue/ValveKeyValue/KVSerializerOptions.cs b/ValveKeyValue/ValveKeyValue/KVSerializerOptions.cs index d3b3c8d6..f58367e5 100644 --- a/ValveKeyValue/ValveKeyValue/KVSerializerOptions.cs +++ b/ValveKeyValue/ValveKeyValue/KVSerializerOptions.cs @@ -27,6 +27,12 @@ public sealed class KVSerializerOptions /// public IIncludedFileLoader FileLoader { get; set; } + + /// + /// Gets or sets the string table used for smaller binary serialization. + /// + public StringTable StringTable { get; set; } + /// /// Gets the default options (used when none are specified). /// diff --git a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs index 6c2dc155..4a5705d3 100644 --- a/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs +++ b/ValveKeyValue/ValveKeyValue/Serialization/KeyValues1/KV1BinarySerializer.cs @@ -1,3 +1,4 @@ +using System.Linq; using System.Text; using ValveKeyValue.Abstraction; using ValveKeyValue.KeyValues1; @@ -6,14 +7,16 @@ namespace ValveKeyValue.Serialization.KeyValues1 { sealed class KV1BinarySerializer : IVisitationListener, IDisposable { - public KV1BinarySerializer(Stream stream) + public KV1BinarySerializer(Stream stream, StringTable stringTable) { Require.NotNull(stream, nameof(stream)); writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true); + this.stringTable = stringTable; } readonly BinaryWriter writer; + readonly StringTable stringTable; int objectDepth; public void Dispose() @@ -25,7 +28,7 @@ public void OnObjectStart(string name) { objectDepth++; Write(KV1BinaryNodeType.ChildObject); - WriteNullTerminatedBytes(Encoding.UTF8.GetBytes(name)); + WriteKeyForNextValue(name); } public void OnObjectEnd() @@ -42,7 +45,7 @@ public void OnObjectEnd() public void OnKeyValuePair(string name, KVValue value) { Write(GetNodeType(value.ValueType)); - WriteNullTerminatedBytes(Encoding.UTF8.GetBytes(name)); + WriteKeyForNextValue(name); switch (value.ValueType) { @@ -83,6 +86,18 @@ void WriteNullTerminatedBytes(byte[] value) writer.Write((byte)0); } + void WriteKeyForNextValue(string name) + { + if (stringTable is not null) + { + writer.Write(stringTable.GetOrAdd(name)); + } + else + { + WriteNullTerminatedBytes(Encoding.UTF8.GetBytes(name)); + } + } + static KV1BinaryNodeType GetNodeType(KVValueType type) { return type switch diff --git a/ValveKeyValue/ValveKeyValue/StringTable.cs b/ValveKeyValue/ValveKeyValue/StringTable.cs new file mode 100644 index 00000000..0fa13ae4 --- /dev/null +++ b/ValveKeyValue/ValveKeyValue/StringTable.cs @@ -0,0 +1,83 @@ +using System.Linq; + +namespace ValveKeyValue +{ + public sealed class StringTable + { + public StringTable() + : this(new List(), writable: true) + { + } + + public StringTable(int capacity) + : this(new List(capacity), writable: true) + { + } + + public StringTable(IList values) + : this(values, writable: !values.IsReadOnly) + { + } + + StringTable(IList values, bool writable) + { + this.lookup = values; + this.writable = writable; + + reverse = new Dictionary(capacity: lookup.Count, StringComparer.Ordinal); + + for (var i = 0; i < lookup.Count; i++) + { + var value = lookup[i]; + reverse[value] = i; + } + } + + + readonly IList lookup; + readonly bool writable; + readonly Dictionary reverse; + + public string this[int index] + { + get + { + if (index < 0) + { + throw new ArgumentOutOfRangeException(nameof(index), "Index must be non-negative."); + } + + if (index >= lookup.Count) + { + throw new ArgumentOutOfRangeException(nameof(index), index, "Index must be less than the number of strings in the table."); + } + + return lookup[index]; + } + } + + public void Add(string value) + { + if (!writable) + { + throw new InvalidOperationException("Unable to add to read-only string table."); + } + + lookup.Add(value); + reverse.TryAdd(value, lookup.Count - 1); + } + + public int GetOrAdd(string value) + { + if (!reverse.TryGetValue(value, out var index)) + { + Add(value); + index = lookup.Count - 1; + } + + return index; + } + + public string[] ToArray() => lookup.ToArray(); + } +}