diff --git a/TestApp/Program.cs b/TestApp/Program.cs index 168c4ce..4f0c0ed 100644 --- a/TestApp/Program.cs +++ b/TestApp/Program.cs @@ -1,4 +1,5 @@ -using ZeroLevel; +using System; +using ZeroLevel; namespace TestApp { diff --git a/TestApp/TestApp.csproj b/TestApp/TestApp.csproj index c638033..c13cbb7 100644 --- a/TestApp/TestApp.csproj +++ b/TestApp/TestApp.csproj @@ -1,4 +1,4 @@ - + Exe diff --git a/ZeroLevel.SqlServer/ZeroLevel.SqlServer.csproj b/ZeroLevel.SqlServer/ZeroLevel.SqlServer.csproj index a1a1b35..4be0b46 100644 --- a/ZeroLevel.SqlServer/ZeroLevel.SqlServer.csproj +++ b/ZeroLevel.SqlServer/ZeroLevel.SqlServer.csproj @@ -9,7 +9,7 @@ Properties ZeroLevel.SqlServer ZeroLevel.SqlServer - v4.7 + v4.7.2 512 diff --git a/ZeroLevel.UnitTests/SerializationTests.cs b/ZeroLevel.UnitTests/SerializationTests.cs index d4b710e..4a0326f 100644 --- a/ZeroLevel.UnitTests/SerializationTests.cs +++ b/ZeroLevel.UnitTests/SerializationTests.cs @@ -62,6 +62,51 @@ namespace ZeroLevel.Serialization } } + private void MakeDictionaryTest(Dictionary value + , Func keyComparator = null + , Func valueComparator = null) + { + byte[] data; + Dictionary clone; + // Act + using (var writer = new MemoryStreamWriter()) + { + writer.WriteDictionary(value); + data = writer.Complete(); + } + using (var reader = new MemoryStreamReader(data)) + { + clone = reader.ReadDictionary(); + } + + // Assert + if (value == null && clone != null && !clone.Any()) return; // OK + + if (value != null && clone == null) throw new Exception("Fail"); + var original_keys = value.Keys.ToArray(); + var clone_keys = clone.Keys.ToArray(); + + if (keyComparator == null) + { + Assert.True(CollectionComparsionExtensions.NoOrderingEquals(original_keys, clone_keys)); + } + else + { + Assert.True(CollectionComparsionExtensions.NoOrderingEquals(original_keys, clone_keys, keyComparator)); + } + foreach (var key in original_keys) + { + if (valueComparator == null) + { + Assert.Equal(value[key], clone[key]); + } + else + { + Assert.True(valueComparator(value[key], clone[key])); + } + } + } + [Fact] public void SerializeDateTime() { @@ -134,7 +179,20 @@ namespace ZeroLevel.Serialization MakePrimitiveTest("HELLO!", comparator); MakePrimitiveTest("𐌼𐌰𐌲 𐌲𐌻𐌴𐍃 𐌹̈𐍄𐌰𐌽, 𐌽𐌹 𐌼𐌹𐍃 𐍅𐌿 𐌽𐌳𐌰𐌽 𐌱𐍂𐌹𐌲𐌲𐌹𐌸", comparator); } + [Fact] + public void SerizlizeCharText() + { + // Arrange + var line = "abcxyzABCZА-Яа-яёЁйЙ123"; + + // Act + var bytes = line.Select(ch => MessageSerializer.SerializeCompatible(ch)); + + // Assert + var testLine = new string(bytes.Select(ba => MessageSerializer.DeserializeCompatible(ba)).ToArray()); + Assert.Equal(line, testLine); + } [Fact] public void SerializeInt32() { @@ -324,6 +382,24 @@ namespace ZeroLevel.Serialization MakeCollectionTest(new string[] { "", String.Empty, null, "HELLO!", "𐌼𐌰𐌲 𐌲𐌻𐌴𐍃 𐌹̈𐍄𐌰𐌽, 𐌽𐌹 𐌼𐌹𐍃 𐍅𐌿 𐌽𐌳𐌰𐌽 𐌱𐍂𐌹𐌲𐌲𐌹𐌸" }, comparator); } + [Fact] + public void SerizlizeCollectionChar() + { + // Arrange + var line = "abcxyzABCZА-Яа-яёЁйЙ123"; + + // Act + var bytes_string = MessageSerializer.SerializeCompatible(line); + var bytes_charenum = MessageSerializer.SerializeCompatible>(line); + + // Assert + var test_line1 = MessageSerializer.DeserializeCompatible(bytes_string); + var test_line2 = new string(MessageSerializer.DeserializeCompatible>(bytes_charenum).ToArray()); + + Assert.Equal(line, test_line1); + Assert.Equal(line, test_line2); + Assert.NotEqual(bytes_string, bytes_charenum); + } [Fact] public void SerializeCollectionInt32() @@ -403,5 +479,36 @@ namespace ZeroLevel.Serialization Assert.Equal(2049, reader.ReadInt32()); } } + + [Fact] + public void SerializeDictionaryTest() + { + var dict = new Dictionary + { + {0, "Dear" }, + {1, "Chaisy" }, + {2, "Lain" } + }; + MakeDictionaryTest(dict); + } + + [Fact] + public void SerializeDictionaryWithComposedObjectTest() + { + var dict = new Dictionary + { + {0, CompositeInstanceFactory.MakeDocument() }, + {1, CompositeInstanceFactory.MakeDocument() }, + {2, CompositeInstanceFactory.MakeDocument() }, + {3, CompositeInstanceFactory.MakeDocument() } + }; + var comparator = new Func((left, right) => + { + var l_bin = MessageSerializer.Serialize(left); + var r_bin = MessageSerializer.Serialize(right); + return ArrayExtensions.UnsafeEquals(l_bin, r_bin); + }); + MakeDictionaryTest(dict, valueComparator: comparator); + } } } diff --git a/ZeroLevel.UnitTests/ZeroLevel.UnitTests.csproj b/ZeroLevel.UnitTests/ZeroLevel.UnitTests.csproj index 9a1dc8d..45f059c 100644 --- a/ZeroLevel.UnitTests/ZeroLevel.UnitTests.csproj +++ b/ZeroLevel.UnitTests/ZeroLevel.UnitTests.csproj @@ -1,4 +1,4 @@ - + netcoreapp2.2 diff --git a/ZeroLevel/Services/Semantic/Trie.cs b/ZeroLevel/Services/Semantic/Trie.cs index fbdf92d..57b369d 100644 --- a/ZeroLevel/Services/Semantic/Trie.cs +++ b/ZeroLevel/Services/Semantic/Trie.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.Threading; using ZeroLevel.Services.Serialization; @@ -10,16 +11,14 @@ namespace ZeroLevel.Services.Semantic internal class TrieNode : IBinarySerializable { - public char Key; public uint? Value; public TrieNode Parent; - public List Children; + public ConcurrentDictionary Children; public TrieNode() { } public TrieNode(TrieNode parent) { Parent = parent; } public void Deserialize(IBinaryReader reader) { - this.Key = reader.ReadChar(); if (reader.ReadBoolean()) { this.Value = reader.ReadUInt32(); @@ -28,12 +27,11 @@ namespace ZeroLevel.Services.Semantic { this.Value = null; } - this.Children = reader.ReadCollection(); + this.Children = reader.ReadDictionaryAsConcurrent(); } public void Serialize(IBinaryWriter writer) { - writer.WriteChar(this.Key); if (this.Value.HasValue) { writer.WriteBoolean(true); @@ -49,7 +47,7 @@ namespace ZeroLevel.Services.Semantic } else { - writer.WriteCollection(this.Children); + writer.WriteDictionary(this.Children); } } internal TrieNode Append(string word, int index, bool reverse) @@ -65,40 +63,22 @@ namespace ZeroLevel.Services.Semantic { if (this.Children == null) { - this.Children = new List(); + this.Children = new ConcurrentDictionary(); } - for (int i = 0; i < Children.Count; i++) + if (!this.Children.ContainsKey(word[index])) { - if (Children[i].Key == word[index]) - { - return Children[i].Append(word, index + 1, reverse); - } + this.Children.TryAdd(word[index], new TrieNode(this)); } - var tn = new TrieNode(reverse ? this : null) { Key = word[index] }; - Children.Add(tn); - return tn.Append(word, index + 1, reverse); + return Children[word[index]].Append(word, index + 1, reverse); } return null; } internal uint? GetKey(string word, int index) { - if (word.Length == index + 1) + if (this.Children?.ContainsKey(word[index]) ?? false) { - return this.Value; - } - else - { - if (this.Children == null) - { - this.Children = new List(); - } - for (int i = 0; i < Children.Count; i++) - { - if (Children[i].Key == word[index]) - { - return Children[i].GetKey(word, index + 1); - } - } + if (word.Length == index + 1) return this.Children[word[index]].Value; + return this.Children[word[index]].GetKey(word, index + 1); } return null; } @@ -114,7 +94,7 @@ namespace ZeroLevel.Services.Semantic { foreach (var child in this.Children) { - child.RebuildReverseIndex(this, index); + child.Value.RebuildReverseIndex(this, index); } } } @@ -126,17 +106,22 @@ namespace ZeroLevel.Services.Semantic { foreach (var child in this.Children) { - child.DestroyReverseIndex(); + child.Value.DestroyReverseIndex(); } } } } - internal List _roots; + internal TrieNode _root; private int _word_index = 0; private bool _use_reverse_index; private Dictionary _reverse_index; + + public Trie() : this(false) + { + } + public Trie(bool reverse_index = false) { _use_reverse_index = reverse_index; @@ -144,7 +129,7 @@ namespace ZeroLevel.Services.Semantic { _reverse_index = new Dictionary(); } - _roots = new List(); + _root = new TrieNode(); } public void ToggleReverseIndex(bool enabled) @@ -163,37 +148,13 @@ namespace ZeroLevel.Services.Semantic public void Append(string word) { if (word.Length == 0) return; - bool found = false; - for (int i = 0; i < _roots.Count; i++) - { - if (_roots[i].Key == word[0]) - { - var node = _roots[i].Append(word, 1, _use_reverse_index); - Thread.MemoryBarrier(); - if (node != null) - { - node.Value = (uint)Interlocked.Increment(ref _word_index); - if (_use_reverse_index) - { - _reverse_index.Add(node.Value.Value, node); - } - } - found = true; - } - } - if (!found) + var node = _root.Append(word, 0, _use_reverse_index); + if (node != null) { - var tn = new TrieNode { Key = word[0] }; - _roots.Add(tn); - var node = tn.Append(word, 1, _use_reverse_index); - Thread.MemoryBarrier(); node.Value = (uint)Interlocked.Increment(ref _word_index); - if (node != null) + if (_use_reverse_index) { - if (_use_reverse_index) - { - _reverse_index.Add(node.Value.Value, node); - } + _reverse_index.Add(node.Value.Value, node); } } } @@ -201,55 +162,27 @@ namespace ZeroLevel.Services.Semantic public uint? Key(string word) { if (word?.Length == 0) return null; - for (int i = 0; i < _roots.Count; i++) - { - if (_roots[i].Key == word[0]) - { - if (word.Length == 1) - { - return _roots[i].Value; - } - else - { - return _roots[i].GetKey(word, 1); - } - } - } - return null; + return _root.GetKey(word, 0); } public bool Contains(string word) { if (word?.Length == 0) return false; - for (int i = 0; i < _roots.Count; i++) - { - if (_roots[i].Key == word[0]) - { - if (word.Length == 1) - { - return _roots[i].Value.HasValue; - } - else - { - return _roots[i].GetKey(word, 1).HasValue; - } - } - } - return false; + return _root.GetKey(word, 0).HasValue; } public void Serialize(IBinaryWriter writer) { writer.WriteInt32(this._word_index); writer.WriteBoolean(this._use_reverse_index); - writer.WriteCollection(this._roots); + writer.Write(this._root); } public void Deserialize(IBinaryReader reader) { this._word_index = reader.ReadInt32(); this._use_reverse_index = reader.ReadBoolean(); - this._roots = reader.ReadCollection(); + this._root = reader.Read(); RebuildReverseIndex(); } @@ -261,10 +194,7 @@ namespace ZeroLevel.Services.Semantic { _reverse_index = new Dictionary(); } - foreach (var node in _roots) - { - node.RebuildReverseIndex(null, _reverse_index); - } + _root.RebuildReverseIndex(null, _reverse_index); } } @@ -275,10 +205,7 @@ namespace ZeroLevel.Services.Semantic _reverse_index.Clear(); _reverse_index = null; } - foreach (var node in _roots) - { - node.DestroyReverseIndex(); - } + _root.DestroyReverseIndex(); } } } diff --git a/ZeroLevel/Services/Serialization/IBinaryReader.cs b/ZeroLevel/Services/Serialization/IBinaryReader.cs index 8235d38..cd3b538 100644 --- a/ZeroLevel/Services/Serialization/IBinaryReader.cs +++ b/ZeroLevel/Services/Serialization/IBinaryReader.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Net; @@ -52,12 +53,16 @@ namespace ZeroLevel.Services.Serialization List ReadCollection() where T : IBinarySerializable, new(); + Dictionary ReadDictionary(); + + ConcurrentDictionary ReadDictionaryAsConcurrent(); + List ReadStringCollection(); List ReadGuidCollection(); List ReadDateTimeCollection(); - + List ReadCharCollection(); List ReadInt64Collection(); List ReadInt32Collection(); diff --git a/ZeroLevel/Services/Serialization/IBinaryWriter.cs b/ZeroLevel/Services/Serialization/IBinaryWriter.cs index 8d2609c..d92764f 100644 --- a/ZeroLevel/Services/Serialization/IBinaryWriter.cs +++ b/ZeroLevel/Services/Serialization/IBinaryWriter.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Net; @@ -51,7 +52,12 @@ namespace ZeroLevel.Services.Serialization void WriteCollection(IEnumerable collection) where T : IBinarySerializable; + void WriteDictionary(Dictionary collection); + void WriteDictionary(ConcurrentDictionary collection); + + void WriteCollection(IEnumerable collection); + void WriteCollection(IEnumerable collection); void WriteCollection(IEnumerable collection); diff --git a/ZeroLevel/Services/Serialization/MemoryStreamReader.cs b/ZeroLevel/Services/Serialization/MemoryStreamReader.cs index 65de261..e742b45 100644 --- a/ZeroLevel/Services/Serialization/MemoryStreamReader.cs +++ b/ZeroLevel/Services/Serialization/MemoryStreamReader.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Net; @@ -54,10 +55,10 @@ namespace ZeroLevel.Services.Serialization throw new OutOfMemoryException("Array index out of bounds"); return (byte)_stream.ReadByte(); } - + public char ReadChar() { - if (CheckOutOfRange(_stream, 1)) + if (CheckOutOfRange(_stream, 2)) throw new OutOfMemoryException("Array index out of bounds"); var buffer = ReadBuffer(2); return BitConverter.ToChar(buffer, 0); @@ -240,6 +241,43 @@ namespace ZeroLevel.Services.Serialization return collection; } + + public Dictionary ReadDictionary() + { + int count = ReadInt32(); + var collection = new Dictionary(count); + if (count > 0) + { + TKey key; + TValue value; + for (int i = 0; i < count; i++) + { + key = ReadCompatible(); + value = ReadCompatible(); + collection.Add(key, value); + } + } + return collection; + } + + public ConcurrentDictionary ReadDictionaryAsConcurrent() + { + int count = ReadInt32(); + var collection = new ConcurrentDictionary(); + if (count > 0) + { + TKey key; + TValue value; + for (int i = 0; i < count; i++) + { + key = ReadCompatible(); + value = ReadCompatible(); + collection.TryAdd(key, value); + } + } + return collection; + } + public T ReadCompatible() { return MessageSerializer.DeserializeCompatible(this); @@ -379,7 +417,19 @@ namespace ZeroLevel.Services.Serialization } return collection; } - + public List ReadCharCollection() + { + int count = ReadInt32(); + var collection = new List(count); + if (count > 0) + { + for (int i = 0; i < count; i++) + { + collection.Add(ReadChar()); + } + } + return collection; + } public List ReadShortCollection() { int count = ReadInt32(); diff --git a/ZeroLevel/Services/Serialization/MemoryStreamWriter.cs b/ZeroLevel/Services/Serialization/MemoryStreamWriter.cs index cc78595..0cbad0d 100644 --- a/ZeroLevel/Services/Serialization/MemoryStreamWriter.cs +++ b/ZeroLevel/Services/Serialization/MemoryStreamWriter.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -50,7 +51,8 @@ namespace ZeroLevel.Services.Serialization /// public void WriteChar(char val) { - _stream.Write(BitConverter.GetBytes(val), 0, 2); + var data = BitConverter.GetBytes(val); + _stream.Write(data, 0, 2); } /// @@ -307,6 +309,17 @@ namespace ZeroLevel.Services.Serialization } } + public void WriteCollection(IEnumerable collection) + { + WriteInt32(collection?.Count() ?? 0); + if (collection != null) + { + foreach (var item in collection) + { + WriteChar(item); + } + } + } public void WriteCollection(IEnumerable collection) { WriteInt32(collection?.Count() ?? 0); @@ -417,7 +430,8 @@ namespace ZeroLevel.Services.Serialization public void WriteCompatible(T item) { - WriteBytes(MessageSerializer.SerializeCompatible(item)); + var buffer = MessageSerializer.SerializeCompatible(item); + _stream.Write(buffer, 0, buffer.Length); } public void WriteCollection(IEnumerable collection) @@ -432,6 +446,33 @@ namespace ZeroLevel.Services.Serialization } } + + public void WriteDictionary(Dictionary collection) + { + WriteInt32(collection?.Count() ?? 0); + if (collection != null) + { + foreach (var item in collection) + { + WriteCompatible(item.Key); + WriteCompatible(item.Value); + } + } + } + + public void WriteDictionary(ConcurrentDictionary collection) + { + WriteInt32(collection?.Count() ?? 0); + if (collection != null) + { + foreach (var item in collection) + { + WriteCompatible(item.Key); + WriteCompatible(item.Value); + } + } + } + public void WriteCollection(IEnumerable collection) { WriteInt32(collection?.Count() ?? 0); diff --git a/ZeroLevel/Services/Serialization/PrimitiveTypeSerializer.cs b/ZeroLevel/Services/Serialization/PrimitiveTypeSerializer.cs index 1ed2671..36beba0 100644 --- a/ZeroLevel/Services/Serialization/PrimitiveTypeSerializer.cs +++ b/ZeroLevel/Services/Serialization/PrimitiveTypeSerializer.cs @@ -49,6 +49,7 @@ namespace ZeroLevel.Services.Serialization private static void PreloadCachee() { + _cachee.Add(typeof(char), Create()); _cachee.Add(typeof(Boolean), Create()); _cachee.Add(typeof(Byte), Create()); _cachee.Add(typeof(Byte[]), Create()); @@ -68,6 +69,7 @@ namespace ZeroLevel.Services.Serialization _cachee.Add(typeof(IPEndPoint), Create()); _cachee.Add(typeof(IPAddress), Create()); + _cachee.Add(typeof(IEnumerable), Create>()); _cachee.Add(typeof(IEnumerable), Create>()); _cachee.Add(typeof(IEnumerable), Create>()); _cachee.Add(typeof(IEnumerable), Create>()); @@ -87,6 +89,7 @@ namespace ZeroLevel.Services.Serialization _cachee.Add(typeof(IEnumerable), Create>()); _cachee.Add(typeof(IEnumerable), Create>()); + _enumTypesCachee.Add(typeof(char), typeof(IEnumerable)); _enumTypesCachee.Add(typeof(Boolean), typeof(IEnumerable)); _enumTypesCachee.Add(typeof(Byte), typeof(IEnumerable)); _enumTypesCachee.Add(typeof(Byte[]), typeof(IEnumerable)); @@ -121,6 +124,11 @@ namespace ZeroLevel.Services.Serialization wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadUInt32").First(); wrapper.WriteId = wrapper.Invoker.Configure(typeof(MemoryStreamWriter), "WriteUInt32").First(); } + else if (type == typeof(char)) + { + wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadChar").First(); + wrapper.WriteId = wrapper.Invoker.Configure(typeof(MemoryStreamWriter), "WriteChar").First(); + } else if (type == typeof(Boolean)) { wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadBoolean").First(); @@ -209,6 +217,11 @@ namespace ZeroLevel.Services.Serialization wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadInt32Collection").First(); wrapper.WriteId = wrapper.Invoker.Configure(typeof(MemoryStreamWriter), CreatePredicate()).First(); } + else if (type == typeof(IEnumerable)) + { + wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadCharCollection").First(); + wrapper.WriteId = wrapper.Invoker.Configure(typeof(MemoryStreamWriter), CreatePredicate()).First(); + } else if (type == typeof(IEnumerable)) { wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadUInt32Collection").First(); diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index 4cc8295..a300a28 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -2,10 +2,11 @@ netstandard2.0 - Infrastructure layer library + Fixed dictionary serialization, trie + ogoun ogoun - 3.0.0.6 + 3.0.0.7 Added char serialization Added prefix tree (Trie) https://github.com/ogoun/Zero/wiki @@ -14,8 +15,8 @@ Added prefix tree (Trie) https://raw.githubusercontent.com/ogoun/Zero/master/zero.png https://github.com/ogoun/Zero GitHub - 3.0.6 - 3.0.0.6 + 3.0.7 + 3.0.0.7