diff --git a/ZeroLevel.UnitTests/CollectionsTests.cs b/ZeroLevel.UnitTests/CollectionsTests.cs index ad4a77a..6eabd99 100644 --- a/ZeroLevel.UnitTests/CollectionsTests.cs +++ b/ZeroLevel.UnitTests/CollectionsTests.cs @@ -1,4 +1,7 @@ -using System.Linq; +using System; +using System.IO; +using System.Linq; +using System.Net; using Xunit; using ZeroLevel.Services.Collections; @@ -114,5 +117,71 @@ namespace ZeroLevel.CollectionUnitTests Assert.True(CollectionComparsionExtensions.OrderingEquals(chunks_2[3], new long[] { 7, 8 })); Assert.True(CollectionComparsionExtensions.OrderingEquals(chunks_2[4], new long[] { 9 })); } + + [Fact] + public void EverythingStorageTest() + { + // Arrange + var storage = EverythingStorage.Create(); + + // Act + storage.Add("int_a", 101); + storage.Add("int_b", 255); + storage.Add("int_c", 1024); + + storage.Add("ts_a", TimeSpan.FromSeconds(6546)); + storage.Add("ts_b", TimeSpan.FromSeconds(777)); + + storage.Add("l_max", ulong.MaxValue); + + storage.Add("ip", IPAddress.Loopback); + + // Assert + Assert.Equal(101, storage.Get("int_a")); + Assert.Equal(255, storage.Get("int_b")); + Assert.Equal(1024, storage.Get("int_c")); + + Assert.Equal(TimeSpan.FromSeconds(6546), storage.Get("ts_a")); + Assert.Equal(TimeSpan.FromSeconds(777), storage.Get("ts_b")); + + Assert.Equal(ulong.MaxValue, storage.Get("l_max")); + + Assert.Equal(IPAddress.Loopback, storage.Get("ip")); + } + + [Fact] + public void EverythingStorageDumpTest() + { + // Arrange + var storage = EverythingStorage.Create(); + + // Act + storage.Add("int_a", 101); + storage.Add("int_b", 255); + storage.Add("int_c", 1024); + + storage.Add("ts_a", TimeSpan.FromSeconds(6546)); + storage.Add("ts_b", TimeSpan.FromSeconds(777)); + + storage.Add("l_max", ulong.MaxValue); + + var restored = EverythingStorage.Create(); + + var file = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + storage.Save(file); + restored.Load(file); + + File.Delete(file); + + // Assert + Assert.Equal(101, restored.Get("int_a")); + Assert.Equal(255, restored.Get("int_b")); + Assert.Equal(1024, restored.Get("int_c")); + + Assert.Equal(TimeSpan.FromSeconds(6546), restored.Get("ts_a")); + Assert.Equal(TimeSpan.FromSeconds(777), restored.Get("ts_b")); + + Assert.Equal(ulong.MaxValue, restored.Get("l_max")); + } } } diff --git a/ZeroLevel.UnitTests/DependencyInjectionTests.cs b/ZeroLevel.UnitTests/DependencyInjectionTests.cs index 899636c..2f14acf 100644 --- a/ZeroLevel.UnitTests/DependencyInjectionTests.cs +++ b/ZeroLevel.UnitTests/DependencyInjectionTests.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Xunit; +using Xunit; using ZeroLevel.DependencyInjection; namespace ZeroLevel.UnitTests diff --git a/ZeroLevel.UnitTests/SerializationTests.cs b/ZeroLevel.UnitTests/SerializationTests.cs index c7d5e98..b78c0ca 100644 --- a/ZeroLevel.UnitTests/SerializationTests.cs +++ b/ZeroLevel.UnitTests/SerializationTests.cs @@ -403,6 +403,50 @@ namespace ZeroLevel.Serialization Assert.NotEqual(bytes_string, bytes_charenum); } + [Fact] + public void EOSTest() + { + var data = new Dictionary + { + { 0, "asd"}, + { 1, "sdf"}, + { 2, "dfg"}, + { 3, "fgh"}, + { 4, "ghj"} + }; + var num_data = long.MaxValue >> 1; + var date_data = DateTime.UtcNow; + byte[] serialized; + using (var writer = new MemoryStreamWriter()) + { + writer.WriteDateTime(date_data); + foreach (var key in data.Keys.OrderBy(k => k)) + { + writer.WriteLong(key); + writer.WriteString(data[key]); + } + writer.WriteLong(num_data); + serialized = writer.Complete(); + } + using (var reader = new MemoryStreamReader(serialized)) + { + Assert.False(reader.EOS); + var date = reader.ReadDateTime(); + Assert.Equal(date, date_data); + Assert.False(reader.EOS); + for (long i = 0; i < 5; i++) + { + Assert.Equal(i, reader.ReadLong()); + Assert.False(reader.EOS); + Assert.Equal(data[i], reader.ReadString()); + Assert.False(reader.EOS); + } + var num = reader.ReadLong(); + Assert.Equal(num, num_data); + Assert.True(reader.EOS); + } + } + [Fact] public void SerializeCollectionInt32() { diff --git a/ZeroLevel/Services/Collections/EverythingStorage.cs b/ZeroLevel/Services/Collections/EverythingStorage.cs index 1f991d8..a2c88bb 100644 --- a/ZeroLevel/Services/Collections/EverythingStorage.cs +++ b/ZeroLevel/Services/Collections/EverythingStorage.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.IO; using System.Linq; using ZeroLevel.Services.Invokation; +using ZeroLevel.Services.Serialization; namespace ZeroLevel.Services.Collections { @@ -15,6 +17,7 @@ namespace ZeroLevel.Services.Collections } private class ConcreteTypeRepository + : IBinarySerializable { private readonly IInvokeWrapper _wrapper; @@ -24,9 +27,11 @@ namespace ZeroLevel.Services.Collections private readonly Invoker _getter; private readonly Invoker _keys_getter; private readonly object _instance; + private readonly Type _valueType; public ConcreteTypeRepository(Type entityType) { + _valueType = entityType; _wrapper = InvokeWrapper.Create(); var genericType = typeof(Dictionary<,>); var instanceType = genericType.MakeGenericType(new Type[] { typeof(string), entityType }); @@ -46,7 +51,7 @@ namespace ZeroLevel.Services.Collections var get_key = _wrapper.Configure(getter); _getter = _wrapper.GetInvoker(get_key); - var k = instanceType.GetProperty("Keys", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public| System.Reflection.BindingFlags.NonPublic); + var k = instanceType.GetProperty("Keys", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic); var keys_getter = k.GetGetMethod(); var get_keys = _wrapper.Configure(keys_getter); _keys_getter = _wrapper.GetInvoker(get_keys); @@ -88,6 +93,35 @@ namespace ZeroLevel.Services.Collections { return _getter.Invoke(_instance, key); } + + public Type GetEntityType() => _valueType; + + public void Serialize(IBinaryWriter writer) + { + writer.WriteString(_valueType.FullName); + + var keys = Keys().ToArray(); + writer.WriteInt32(keys.Length); + for (int i = 0; i < keys.Length; i++) + { + writer.WriteString(keys[i]); + writer.WriteBytes(MessageSerializer.SerializeCompatible(Get(keys[i]))); + } + } + + public void Deserialize(IBinaryReader reader) + { + var typeName = reader.ReadString(); + var type = Type.GetType(typeName); + + var count = reader.ReadInt32(); + for (int i = 0; i < count; i++) + { + var key = reader.ReadString(); + var val = MessageSerializer.DeserializeCompatible(type, reader.ReadBytes()); + Insert(key, val); + } + } } private readonly ConcurrentDictionary _shardedRepositories = @@ -208,5 +242,50 @@ namespace ZeroLevel.Services.Collections { return this[typeof(T)].Keys(); } + + public void Save(string path) + { + using (var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None)) + { + Save(stream); + } + } + + public void Load(string path) + { + using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None)) + { + Load(stream); + } + } + + public void Save(Stream stream) + { + using (var writer = new MemoryStreamWriter(stream)) + { + var keys = _shardedRepositories.Keys.ToArray(); + writer.WriteInt32(keys.Length); + for (int i = 0; i < keys.Length; i++) + { + writer.WriteString(keys[i].FullName); + writer.Write(_shardedRepositories[keys[i]]); + } + } + } + + public void Load(Stream stream) + { + using (var reader = new MemoryStreamReader(stream)) + { + var count = reader.ReadInt32(); + for (int i = 0; i < count; i++) + { + var typeName = reader.ReadString(); + var type = Type.GetType(typeName); + var rep = reader.Read(type); + _shardedRepositories.TryAdd(rep.GetEntityType(), rep); + } + } + } } } \ No newline at end of file diff --git a/ZeroLevel/Services/Collections/IEverythingStorage.cs b/ZeroLevel/Services/Collections/IEverythingStorage.cs index 607a015..b58dac6 100644 --- a/ZeroLevel/Services/Collections/IEverythingStorage.cs +++ b/ZeroLevel/Services/Collections/IEverythingStorage.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; namespace ZeroLevel.Services.Collections { @@ -36,5 +37,13 @@ namespace ZeroLevel.Services.Collections void Remove(Type type, string key); object Get(Type type, string key); + + void Save(string path); + + void Load(string path); + + void Save(Stream stream); + + void Load(Stream stream); } } \ No newline at end of file diff --git a/ZeroLevel/Services/FileSystem/BigFileParser.cs b/ZeroLevel/Services/FileSystem/BigFileParser.cs index 5a148d8..ba73df7 100644 --- a/ZeroLevel/Services/FileSystem/BigFileParser.cs +++ b/ZeroLevel/Services/FileSystem/BigFileParser.cs @@ -27,6 +27,7 @@ namespace ZeroLevel.Services.FileSystem _filePath = filePath; _parser = parser; _bufferSize = bufferSize; + } public IEnumerable> ReadBatches(int batchSize) diff --git a/ZeroLevel/Services/Logging/ILog.cs b/ZeroLevel/Services/Logging/ILog.cs deleted file mode 100644 index b5f4f23..0000000 --- a/ZeroLevel/Services/Logging/ILog.cs +++ /dev/null @@ -1,52 +0,0 @@ -using System; - -namespace ZeroLevel.Services.Logging -{ - public interface ILog - { - /// - /// Message output as is, without adding a logging level and date - /// - void Raw(string line, params object[] args); - - /// - /// Message - /// - void Info(string line, params object[] args); - - /// - /// Warning - /// - void Warning(string line, params object[] args); - - /// - /// Error - /// - void Error(string line, params object[] args); - - /// - /// Error - /// - void Error(Exception ex, string line, params object[] args); - - /// - /// Fatal crash - /// - void Fatal(string line, params object[] args); - - /// - /// Fatal crash - /// - void Fatal(Exception ex, string line, params object[] args); - - /// - /// Debug info - /// - void Debug(string line, params object[] args); - - /// - /// Low Level Debug info - /// - void Verbose(string line, params object[] args); - } -} \ No newline at end of file diff --git a/ZeroLevel/Services/Logging/Implementation/TextFileLogger.cs b/ZeroLevel/Services/Logging/Implementation/TextFileLogger.cs index d672c28..35127c7 100644 --- a/ZeroLevel/Services/Logging/Implementation/TextFileLogger.cs +++ b/ZeroLevel/Services/Logging/Implementation/TextFileLogger.cs @@ -6,7 +6,6 @@ using System.IO.Compression; using System.Linq; using System.Text; using ZeroLevel.Services.FileSystem; -using ZeroLevel.Services.Logging; namespace ZeroLevel.Logging { diff --git a/ZeroLevel/Services/Serialization/IBinaryReader.cs b/ZeroLevel/Services/Serialization/IBinaryReader.cs index 1a90a7a..0f431f2 100644 --- a/ZeroLevel/Services/Serialization/IBinaryReader.cs +++ b/ZeroLevel/Services/Serialization/IBinaryReader.cs @@ -95,6 +95,7 @@ namespace ZeroLevel.Services.Serialization #endregion T Read() where T : IBinarySerializable; + T Read(object arg) where T : IBinarySerializable; T ReadCompatible(); diff --git a/ZeroLevel/Services/Serialization/MemoryStreamReader.cs b/ZeroLevel/Services/Serialization/MemoryStreamReader.cs index 8898adb..9259889 100644 --- a/ZeroLevel/Services/Serialization/MemoryStreamReader.cs +++ b/ZeroLevel/Services/Serialization/MemoryStreamReader.cs @@ -851,6 +851,15 @@ namespace ZeroLevel.Services.Serialization item.Deserialize(this); return item; } + + public T Read(object arg) where T : IBinarySerializable + { + byte type = ReadByte(); + if (type == 0) return default(T); + var item = (T)Activator.CreateInstance(typeof(T), arg); + item.Deserialize(this); + return item; + } #endregion Extensions public void Dispose() diff --git a/ZeroLevel/Services/Serialization/MemoryStreamWriter.cs b/ZeroLevel/Services/Serialization/MemoryStreamWriter.cs index cb75469..590d5dd 100644 --- a/ZeroLevel/Services/Serialization/MemoryStreamWriter.cs +++ b/ZeroLevel/Services/Serialization/MemoryStreamWriter.cs @@ -796,7 +796,6 @@ namespace ZeroLevel.Services.Serialization } } } - #endregion Extension } } \ No newline at end of file