serialization
pull/1/head
Ogoun 5 years ago
parent 33b61aa7f1
commit d3cc629a8e

@ -1,4 +1,5 @@
using ZeroLevel;
using System;
using ZeroLevel;
namespace TestApp
{

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>

@ -9,7 +9,7 @@
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>ZeroLevel.SqlServer</RootNamespace>
<AssemblyName>ZeroLevel.SqlServer</AssemblyName>
<TargetFrameworkVersion>v4.7</TargetFrameworkVersion>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
</PropertyGroup>

@ -62,6 +62,51 @@ namespace ZeroLevel.Serialization
}
}
private void MakeDictionaryTest<TKey, TValue>(Dictionary<TKey, TValue> value
, Func<TKey, TKey, bool> keyComparator = null
, Func<TValue, TValue, bool> valueComparator = null)
{
byte[] data;
Dictionary<TKey, TValue> clone;
// Act
using (var writer = new MemoryStreamWriter())
{
writer.WriteDictionary<TKey, TValue>(value);
data = writer.Complete();
}
using (var reader = new MemoryStreamReader(data))
{
clone = reader.ReadDictionary<TKey, TValue>();
}
// Assert
if (value == null && clone != null && !clone.Any()) return; // OK
if (value != null && clone == null) throw new Exception("Fail");
var original_keys = value.Keys.ToArray();
var clone_keys = clone.Keys.ToArray();
if (keyComparator == null)
{
Assert.True(CollectionComparsionExtensions.NoOrderingEquals(original_keys, clone_keys));
}
else
{
Assert.True(CollectionComparsionExtensions.NoOrderingEquals(original_keys, clone_keys, keyComparator));
}
foreach (var key in original_keys)
{
if (valueComparator == null)
{
Assert.Equal(value[key], clone[key]);
}
else
{
Assert.True(valueComparator(value[key], clone[key]));
}
}
}
[Fact]
public void SerializeDateTime()
{
@ -134,7 +179,20 @@ namespace ZeroLevel.Serialization
MakePrimitiveTest<String>("HELLO!", comparator);
MakePrimitiveTest<String>("𐌼𐌰𐌲 𐌲𐌻𐌴𐍃 𐌹̈𐍄𐌰𐌽, 𐌽𐌹 𐌼𐌹𐍃 𐍅𐌿 𐌽𐌳𐌰𐌽 𐌱𐍂𐌹𐌲𐌲𐌹𐌸", comparator);
}
[Fact]
public void SerizlizeCharText()
{
// Arrange
var line = "abcxyzABCZА-Яа-яёЁйЙ123";
// Act
var bytes = line.Select(ch => MessageSerializer.SerializeCompatible<char>(ch));
// Assert
var testLine = new string(bytes.Select(ba => MessageSerializer.DeserializeCompatible<char>(ba)).ToArray());
Assert.Equal(line, testLine);
}
[Fact]
public void SerializeInt32()
{
@ -324,6 +382,24 @@ namespace ZeroLevel.Serialization
MakeCollectionTest<String>(new string[] { "", String.Empty, null, "HELLO!", "𐌼𐌰𐌲 𐌲𐌻𐌴𐍃 𐌹̈𐍄𐌰𐌽, 𐌽𐌹 𐌼𐌹𐍃 𐍅𐌿 𐌽𐌳𐌰𐌽 𐌱𐍂𐌹𐌲𐌲𐌹𐌸" }, comparator);
}
[Fact]
public void SerizlizeCollectionChar()
{
// Arrange
var line = "abcxyzABCZА-Яа-яёЁйЙ123";
// Act
var bytes_string = MessageSerializer.SerializeCompatible<string>(line);
var bytes_charenum = MessageSerializer.SerializeCompatible<IEnumerable<char>>(line);
// Assert
var test_line1 = MessageSerializer.DeserializeCompatible<string>(bytes_string);
var test_line2 = new string(MessageSerializer.DeserializeCompatible<IEnumerable<char>>(bytes_charenum).ToArray());
Assert.Equal(line, test_line1);
Assert.Equal(line, test_line2);
Assert.NotEqual(bytes_string, bytes_charenum);
}
[Fact]
public void SerializeCollectionInt32()
@ -403,5 +479,36 @@ namespace ZeroLevel.Serialization
Assert.Equal(2049, reader.ReadInt32());
}
}
[Fact]
public void SerializeDictionaryTest()
{
var dict = new Dictionary<int, string>
{
{0, "Dear" },
{1, "Chaisy" },
{2, "Lain" }
};
MakeDictionaryTest(dict);
}
[Fact]
public void SerializeDictionaryWithComposedObjectTest()
{
var dict = new Dictionary<int, Document>
{
{0, CompositeInstanceFactory.MakeDocument() },
{1, CompositeInstanceFactory.MakeDocument() },
{2, CompositeInstanceFactory.MakeDocument() },
{3, CompositeInstanceFactory.MakeDocument() }
};
var comparator = new Func<Document, Document, bool>((left, right) =>
{
var l_bin = MessageSerializer.Serialize(left);
var r_bin = MessageSerializer.Serialize(right);
return ArrayExtensions.UnsafeEquals(l_bin, r_bin);
});
MakeDictionaryTest(dict, valueComparator: comparator);
}
}
}

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp2.2</TargetFramework>

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using ZeroLevel.Services.Serialization;
@ -10,16 +11,14 @@ namespace ZeroLevel.Services.Semantic
internal class TrieNode
: IBinarySerializable
{
public char Key;
public uint? Value;
public TrieNode Parent;
public List<TrieNode> Children;
public ConcurrentDictionary<char, TrieNode> Children;
public TrieNode() { }
public TrieNode(TrieNode parent) { Parent = parent; }
public void Deserialize(IBinaryReader reader)
{
this.Key = reader.ReadChar();
if (reader.ReadBoolean())
{
this.Value = reader.ReadUInt32();
@ -28,12 +27,11 @@ namespace ZeroLevel.Services.Semantic
{
this.Value = null;
}
this.Children = reader.ReadCollection<TrieNode>();
this.Children = reader.ReadDictionaryAsConcurrent<char, TrieNode>();
}
public void Serialize(IBinaryWriter writer)
{
writer.WriteChar(this.Key);
if (this.Value.HasValue)
{
writer.WriteBoolean(true);
@ -49,7 +47,7 @@ namespace ZeroLevel.Services.Semantic
}
else
{
writer.WriteCollection<TrieNode>(this.Children);
writer.WriteDictionary<char, TrieNode>(this.Children);
}
}
internal TrieNode Append(string word, int index, bool reverse)
@ -65,40 +63,22 @@ namespace ZeroLevel.Services.Semantic
{
if (this.Children == null)
{
this.Children = new List<TrieNode>();
this.Children = new ConcurrentDictionary<char, TrieNode>();
}
for (int i = 0; i < Children.Count; i++)
if (!this.Children.ContainsKey(word[index]))
{
if (Children[i].Key == word[index])
{
return Children[i].Append(word, index + 1, reverse);
}
this.Children.TryAdd(word[index], new TrieNode(this));
}
var tn = new TrieNode(reverse ? this : null) { Key = word[index] };
Children.Add(tn);
return tn.Append(word, index + 1, reverse);
return Children[word[index]].Append(word, index + 1, reverse);
}
return null;
}
internal uint? GetKey(string word, int index)
{
if (word.Length == index + 1)
if (this.Children?.ContainsKey(word[index]) ?? false)
{
return this.Value;
}
else
{
if (this.Children == null)
{
this.Children = new List<TrieNode>();
}
for (int i = 0; i < Children.Count; i++)
{
if (Children[i].Key == word[index])
{
return Children[i].GetKey(word, index + 1);
}
}
if (word.Length == index + 1) return this.Children[word[index]].Value;
return this.Children[word[index]].GetKey(word, index + 1);
}
return null;
}
@ -114,7 +94,7 @@ namespace ZeroLevel.Services.Semantic
{
foreach (var child in this.Children)
{
child.RebuildReverseIndex(this, index);
child.Value.RebuildReverseIndex(this, index);
}
}
}
@ -126,17 +106,22 @@ namespace ZeroLevel.Services.Semantic
{
foreach (var child in this.Children)
{
child.DestroyReverseIndex();
child.Value.DestroyReverseIndex();
}
}
}
}
internal List<TrieNode> _roots;
internal TrieNode _root;
private int _word_index = 0;
private bool _use_reverse_index;
private Dictionary<uint, TrieNode> _reverse_index;
public Trie() : this(false)
{
}
public Trie(bool reverse_index = false)
{
_use_reverse_index = reverse_index;
@ -144,7 +129,7 @@ namespace ZeroLevel.Services.Semantic
{
_reverse_index = new Dictionary<uint, TrieNode>();
}
_roots = new List<TrieNode>();
_root = new TrieNode();
}
public void ToggleReverseIndex(bool enabled)
@ -163,37 +148,13 @@ namespace ZeroLevel.Services.Semantic
public void Append(string word)
{
if (word.Length == 0) return;
bool found = false;
for (int i = 0; i < _roots.Count; i++)
{
if (_roots[i].Key == word[0])
{
var node = _roots[i].Append(word, 1, _use_reverse_index);
Thread.MemoryBarrier();
if (node != null)
{
node.Value = (uint)Interlocked.Increment(ref _word_index);
if (_use_reverse_index)
{
_reverse_index.Add(node.Value.Value, node);
}
}
found = true;
}
}
if (!found)
var node = _root.Append(word, 0, _use_reverse_index);
if (node != null)
{
var tn = new TrieNode { Key = word[0] };
_roots.Add(tn);
var node = tn.Append(word, 1, _use_reverse_index);
Thread.MemoryBarrier();
node.Value = (uint)Interlocked.Increment(ref _word_index);
if (node != null)
if (_use_reverse_index)
{
if (_use_reverse_index)
{
_reverse_index.Add(node.Value.Value, node);
}
_reverse_index.Add(node.Value.Value, node);
}
}
}
@ -201,55 +162,27 @@ namespace ZeroLevel.Services.Semantic
public uint? Key(string word)
{
if (word?.Length == 0) return null;
for (int i = 0; i < _roots.Count; i++)
{
if (_roots[i].Key == word[0])
{
if (word.Length == 1)
{
return _roots[i].Value;
}
else
{
return _roots[i].GetKey(word, 1);
}
}
}
return null;
return _root.GetKey(word, 0);
}
public bool Contains(string word)
{
if (word?.Length == 0) return false;
for (int i = 0; i < _roots.Count; i++)
{
if (_roots[i].Key == word[0])
{
if (word.Length == 1)
{
return _roots[i].Value.HasValue;
}
else
{
return _roots[i].GetKey(word, 1).HasValue;
}
}
}
return false;
return _root.GetKey(word, 0).HasValue;
}
public void Serialize(IBinaryWriter writer)
{
writer.WriteInt32(this._word_index);
writer.WriteBoolean(this._use_reverse_index);
writer.WriteCollection<TrieNode>(this._roots);
writer.Write<TrieNode>(this._root);
}
public void Deserialize(IBinaryReader reader)
{
this._word_index = reader.ReadInt32();
this._use_reverse_index = reader.ReadBoolean();
this._roots = reader.ReadCollection<TrieNode>();
this._root = reader.Read<TrieNode>();
RebuildReverseIndex();
}
@ -261,10 +194,7 @@ namespace ZeroLevel.Services.Semantic
{
_reverse_index = new Dictionary<uint, TrieNode>();
}
foreach (var node in _roots)
{
node.RebuildReverseIndex(null, _reverse_index);
}
_root.RebuildReverseIndex(null, _reverse_index);
}
}
@ -275,10 +205,7 @@ namespace ZeroLevel.Services.Semantic
_reverse_index.Clear();
_reverse_index = null;
}
foreach (var node in _roots)
{
node.DestroyReverseIndex();
}
_root.DestroyReverseIndex();
}
}
}

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Net;
@ -52,12 +53,16 @@ namespace ZeroLevel.Services.Serialization
List<T> ReadCollection<T>() where T : IBinarySerializable, new();
Dictionary<TKey, TValue> ReadDictionary<TKey, TValue>();
ConcurrentDictionary<TKey, TValue> ReadDictionaryAsConcurrent<TKey, TValue>();
List<string> ReadStringCollection();
List<Guid> ReadGuidCollection();
List<DateTime> ReadDateTimeCollection();
List<char> ReadCharCollection();
List<Int64> ReadInt64Collection();
List<Int32> ReadInt32Collection();

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Net;
@ -51,7 +52,12 @@ namespace ZeroLevel.Services.Serialization
void WriteCollection<T>(IEnumerable<T> collection)
where T : IBinarySerializable;
void WriteDictionary<TKey, TValue>(Dictionary<TKey, TValue> collection);
void WriteDictionary<TKey, TValue>(ConcurrentDictionary<TKey, TValue> collection);
void WriteCollection(IEnumerable<string> collection);
void WriteCollection(IEnumerable<char> collection);
void WriteCollection(IEnumerable<Guid> collection);

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Net;
@ -54,10 +55,10 @@ namespace ZeroLevel.Services.Serialization
throw new OutOfMemoryException("Array index out of bounds");
return (byte)_stream.ReadByte();
}
public char ReadChar()
{
if (CheckOutOfRange(_stream, 1))
if (CheckOutOfRange(_stream, 2))
throw new OutOfMemoryException("Array index out of bounds");
var buffer = ReadBuffer(2);
return BitConverter.ToChar(buffer, 0);
@ -240,6 +241,43 @@ namespace ZeroLevel.Services.Serialization
return collection;
}
public Dictionary<TKey, TValue> ReadDictionary<TKey, TValue>()
{
int count = ReadInt32();
var collection = new Dictionary<TKey, TValue>(count);
if (count > 0)
{
TKey key;
TValue value;
for (int i = 0; i < count; i++)
{
key = ReadCompatible<TKey>();
value = ReadCompatible<TValue>();
collection.Add(key, value);
}
}
return collection;
}
public ConcurrentDictionary<TKey, TValue> ReadDictionaryAsConcurrent<TKey, TValue>()
{
int count = ReadInt32();
var collection = new ConcurrentDictionary<TKey, TValue>();
if (count > 0)
{
TKey key;
TValue value;
for (int i = 0; i < count; i++)
{
key = ReadCompatible<TKey>();
value = ReadCompatible<TValue>();
collection.TryAdd(key, value);
}
}
return collection;
}
public T ReadCompatible<T>()
{
return MessageSerializer.DeserializeCompatible<T>(this);
@ -379,7 +417,19 @@ namespace ZeroLevel.Services.Serialization
}
return collection;
}
public List<char> ReadCharCollection()
{
int count = ReadInt32();
var collection = new List<char>(count);
if (count > 0)
{
for (int i = 0; i < count; i++)
{
collection.Add(ReadChar());
}
}
return collection;
}
public List<short> ReadShortCollection()
{
int count = ReadInt32();

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@ -50,7 +51,8 @@ namespace ZeroLevel.Services.Serialization
/// </summary>
public void WriteChar(char val)
{
_stream.Write(BitConverter.GetBytes(val), 0, 2);
var data = BitConverter.GetBytes(val);
_stream.Write(data, 0, 2);
}
/// <summary>
@ -307,6 +309,17 @@ namespace ZeroLevel.Services.Serialization
}
}
public void WriteCollection(IEnumerable<char> collection)
{
WriteInt32(collection?.Count() ?? 0);
if (collection != null)
{
foreach (var item in collection)
{
WriteChar(item);
}
}
}
public void WriteCollection(IEnumerable<short> collection)
{
WriteInt32(collection?.Count() ?? 0);
@ -417,7 +430,8 @@ namespace ZeroLevel.Services.Serialization
public void WriteCompatible<T>(T item)
{
WriteBytes(MessageSerializer.SerializeCompatible(item));
var buffer = MessageSerializer.SerializeCompatible(item);
_stream.Write(buffer, 0, buffer.Length);
}
public void WriteCollection(IEnumerable<decimal> collection)
@ -432,6 +446,33 @@ namespace ZeroLevel.Services.Serialization
}
}
public void WriteDictionary<TKey, TValue>(Dictionary<TKey, TValue> collection)
{
WriteInt32(collection?.Count() ?? 0);
if (collection != null)
{
foreach (var item in collection)
{
WriteCompatible(item.Key);
WriteCompatible(item.Value);
}
}
}
public void WriteDictionary<TKey, TValue>(ConcurrentDictionary<TKey, TValue> collection)
{
WriteInt32(collection?.Count() ?? 0);
if (collection != null)
{
foreach (var item in collection)
{
WriteCompatible(item.Key);
WriteCompatible(item.Value);
}
}
}
public void WriteCollection(IEnumerable<TimeSpan> collection)
{
WriteInt32(collection?.Count() ?? 0);

@ -49,6 +49,7 @@ namespace ZeroLevel.Services.Serialization
private static void PreloadCachee()
{
_cachee.Add(typeof(char), Create<char>());
_cachee.Add(typeof(Boolean), Create<Boolean>());
_cachee.Add(typeof(Byte), Create<Byte>());
_cachee.Add(typeof(Byte[]), Create<Byte[]>());
@ -68,6 +69,7 @@ namespace ZeroLevel.Services.Serialization
_cachee.Add(typeof(IPEndPoint), Create<IPEndPoint>());
_cachee.Add(typeof(IPAddress), Create<IPAddress>());
_cachee.Add(typeof(IEnumerable<char>), Create<IEnumerable<char>>());
_cachee.Add(typeof(IEnumerable<Boolean>), Create<IEnumerable<Boolean>>());
_cachee.Add(typeof(IEnumerable<Byte>), Create<IEnumerable<Byte>>());
_cachee.Add(typeof(IEnumerable<Byte[]>), Create<IEnumerable<Byte[]>>());
@ -87,6 +89,7 @@ namespace ZeroLevel.Services.Serialization
_cachee.Add(typeof(IEnumerable<IPEndPoint>), Create<IEnumerable<IPEndPoint>>());
_cachee.Add(typeof(IEnumerable<IPAddress>), Create<IEnumerable<IPAddress>>());
_enumTypesCachee.Add(typeof(char), typeof(IEnumerable<char>));
_enumTypesCachee.Add(typeof(Boolean), typeof(IEnumerable<Boolean>));
_enumTypesCachee.Add(typeof(Byte), typeof(IEnumerable<Byte>));
_enumTypesCachee.Add(typeof(Byte[]), typeof(IEnumerable<Byte[]>));
@ -121,6 +124,11 @@ namespace ZeroLevel.Services.Serialization
wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadUInt32").First();
wrapper.WriteId = wrapper.Invoker.Configure(typeof(MemoryStreamWriter), "WriteUInt32").First();
}
else if (type == typeof(char))
{
wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadChar").First();
wrapper.WriteId = wrapper.Invoker.Configure(typeof(MemoryStreamWriter), "WriteChar").First();
}
else if (type == typeof(Boolean))
{
wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadBoolean").First();
@ -209,6 +217,11 @@ namespace ZeroLevel.Services.Serialization
wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadInt32Collection").First();
wrapper.WriteId = wrapper.Invoker.Configure(typeof(MemoryStreamWriter), CreatePredicate<Tw>()).First();
}
else if (type == typeof(IEnumerable<char>))
{
wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadCharCollection").First();
wrapper.WriteId = wrapper.Invoker.Configure(typeof(MemoryStreamWriter), CreatePredicate<Tw>()).First();
}
else if (type == typeof(IEnumerable<UInt32>))
{
wrapper.ReadId = wrapper.Invoker.Configure(typeof(MemoryStreamReader), "ReadUInt32Collection").First();

@ -2,10 +2,11 @@
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Description>Infrastructure layer library</Description>
<Description>Fixed dictionary serialization, trie
</Description>
<Authors>ogoun</Authors>
<Company>ogoun</Company>
<AssemblyVersion>3.0.0.6</AssemblyVersion>
<AssemblyVersion>3.0.0.7</AssemblyVersion>
<PackageReleaseNotes>Added char serialization
Added prefix tree (Trie)</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
@ -14,8 +15,8 @@ Added prefix tree (Trie)</PackageReleaseNotes>
<PackageIconUrl>https://raw.githubusercontent.com/ogoun/Zero/master/zero.png</PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>GitHub</RepositoryType>
<Version>3.0.6</Version>
<FileVersion>3.0.0.6</FileVersion>
<Version>3.0.7</Version>
<FileVersion>3.0.0.7</FileVersion>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">

Loading…
Cancel
Save

Powered by TurnKey Linux.