Refactoring

ZeroLevel.UnitTests/CollectionsTests.cs

@@ -1,4 +1,7 @@
-using System.Linq;
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
 using Xunit;
 using ZeroLevel.Services.Collections;
 
@@ -114,5 +117,71 @@ namespace ZeroLevel.CollectionUnitTests
             Assert.True(CollectionComparsionExtensions.OrderingEquals(chunks_2[3], new long[] { 7, 8 }));
             Assert.True(CollectionComparsionExtensions.OrderingEquals(chunks_2[4], new long[] { 9 }));
         }
+
+        [Fact]
+        public void EverythingStorageTest()
+        {
+            // Arrange
+            var storage = EverythingStorage.Create();
+
+            // Act
+            storage.Add<int>("int_a", 101);
+            storage.Add<int>("int_b", 255);
+            storage.Add<int>("int_c", 1024);
+
+            storage.Add<TimeSpan>("ts_a", TimeSpan.FromSeconds(6546));
+            storage.Add<TimeSpan>("ts_b", TimeSpan.FromSeconds(777));
+
+            storage.Add<ulong>("l_max", ulong.MaxValue);
+
+            storage.Add<IPAddress>("ip", IPAddress.Loopback);
+
+            // Assert
+            Assert.Equal<int>(101, storage.Get<int>("int_a"));
+            Assert.Equal<int>(255, storage.Get<int>("int_b"));
+            Assert.Equal<int>(1024, storage.Get<int>("int_c"));
+
+            Assert.Equal<TimeSpan>(TimeSpan.FromSeconds(6546), storage.Get<TimeSpan>("ts_a"));
+            Assert.Equal<TimeSpan>(TimeSpan.FromSeconds(777), storage.Get<TimeSpan>("ts_b"));
+
+            Assert.Equal<ulong>(ulong.MaxValue, storage.Get<ulong>("l_max"));
+
+            Assert.Equal<IPAddress>(IPAddress.Loopback, storage.Get<IPAddress>("ip"));
+        }
+
+        [Fact]
+        public void EverythingStorageDumpTest()
+        {
+            // Arrange
+            var storage = EverythingStorage.Create();
+
+            // Act
+            storage.Add<int>("int_a", 101);
+            storage.Add<int>("int_b", 255);
+            storage.Add<int>("int_c", 1024);
+
+            storage.Add<TimeSpan>("ts_a", TimeSpan.FromSeconds(6546));
+            storage.Add<TimeSpan>("ts_b", TimeSpan.FromSeconds(777));
+
+            storage.Add<ulong>("l_max", ulong.MaxValue);
+
+            var restored = EverythingStorage.Create();
+
+            var file = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+            storage.Save(file);
+            restored.Load(file);
+
+            File.Delete(file);
+
+            // Assert
+            Assert.Equal<int>(101, restored.Get<int>("int_a"));
+            Assert.Equal<int>(255, restored.Get<int>("int_b"));
+            Assert.Equal<int>(1024, restored.Get<int>("int_c"));
+
+            Assert.Equal<TimeSpan>(TimeSpan.FromSeconds(6546), restored.Get<TimeSpan>("ts_a"));
+            Assert.Equal<TimeSpan>(TimeSpan.FromSeconds(777), restored.Get<TimeSpan>("ts_b"));
+
+            Assert.Equal<ulong>(ulong.MaxValue, restored.Get<ulong>("l_max"));
+        }
     }
 }

ZeroLevel.UnitTests/DependencyInjectionTests.cs

@@ -1,7 +1,4 @@
-using System;
+using Xunit;
-using System.Collections.Generic;
-using System.Text;
-using Xunit;
 using ZeroLevel.DependencyInjection;
 
 namespace ZeroLevel.UnitTests

ZeroLevel.UnitTests/SerializationTests.cs

@@ -403,6 +403,50 @@ namespace ZeroLevel.Serialization
             Assert.NotEqual(bytes_string, bytes_charenum);
         }
 
+        [Fact]
+        public void EOSTest()
+        {
+            var data = new Dictionary<long, string> 
+            {
+                { 0,  "asd"},
+                { 1,  "sdf"},
+                { 2,  "dfg"},
+                { 3,  "fgh"},
+                { 4,  "ghj"}
+            };
+            var num_data = long.MaxValue >> 1;
+            var date_data = DateTime.UtcNow;
+            byte[] serialized;
+            using (var writer = new MemoryStreamWriter())
+            {
+                writer.WriteDateTime(date_data);
+                foreach (var key in data.Keys.OrderBy(k => k))
+                {
+                    writer.WriteLong(key);
+                    writer.WriteString(data[key]);
+                }
+                writer.WriteLong(num_data);
+                serialized = writer.Complete();
+            }
+            using (var reader = new MemoryStreamReader(serialized))
+            {
+                Assert.False(reader.EOS);
+                var date = reader.ReadDateTime();
+                Assert.Equal(date, date_data);
+                Assert.False(reader.EOS);
+                for (long i = 0; i < 5; i++)
+                {
+                    Assert.Equal(i, reader.ReadLong());
+                    Assert.False(reader.EOS);
+                    Assert.Equal(data[i], reader.ReadString());
+                    Assert.False(reader.EOS);
+                }
+                var num = reader.ReadLong();
+                Assert.Equal(num, num_data);
+                Assert.True(reader.EOS);
+            }
+        }
+
         [Fact]
         public void SerializeCollectionInt32()
         {

ZeroLevel/Services/Collections/EverythingStorage.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using ZeroLevel.Services.Invokation;
+using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel.Services.Collections
 {
@@ -15,6 +17,7 @@ namespace ZeroLevel.Services.Collections
         }
 
         private class ConcreteTypeRepository
+            : IBinarySerializable
         {
             private readonly IInvokeWrapper _wrapper;
 
@@ -24,9 +27,11 @@ namespace ZeroLevel.Services.Collections
             private readonly Invoker _getter;
             private readonly Invoker _keys_getter;
             private readonly object _instance;
+            private readonly Type _valueType;
 
             public ConcreteTypeRepository(Type entityType)
             {
+                _valueType = entityType;
                 _wrapper = InvokeWrapper.Create();
                 var genericType = typeof(Dictionary<,>);
                 var instanceType = genericType.MakeGenericType(new Type[] { typeof(string), entityType });
@@ -46,7 +51,7 @@ namespace ZeroLevel.Services.Collections
                 var get_key = _wrapper.Configure(getter);
                 _getter = _wrapper.GetInvoker(get_key);
 
-                var k = instanceType.GetProperty("Keys", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public| System.Reflection.BindingFlags.NonPublic);
+                var k = instanceType.GetProperty("Keys", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic);
                 var keys_getter = k.GetGetMethod();
                 var get_keys = _wrapper.Configure(keys_getter);
                 _keys_getter = _wrapper.GetInvoker(get_keys);
@@ -88,6 +93,35 @@ namespace ZeroLevel.Services.Collections
             {
                 return _getter.Invoke(_instance, key);
             }
+
+            public Type GetEntityType() => _valueType;
+
+            public void Serialize(IBinaryWriter writer)
+            {
+                writer.WriteString(_valueType.FullName);
+
+                var keys = Keys().ToArray();
+                writer.WriteInt32(keys.Length);
+                for (int i = 0; i < keys.Length; i++)
+                {
+                    writer.WriteString(keys[i]);
+                    writer.WriteBytes(MessageSerializer.SerializeCompatible(Get(keys[i])));
+                }
+            }
+
+            public void Deserialize(IBinaryReader reader)
+            {
+                var typeName = reader.ReadString();
+                var type = Type.GetType(typeName);
+
+                var count = reader.ReadInt32();
+                for (int i = 0; i < count; i++)
+                {
+                    var key = reader.ReadString();
+                    var val = MessageSerializer.DeserializeCompatible(type, reader.ReadBytes());
+                    Insert(key, val);
+                }
+            }
         }
 
         private readonly ConcurrentDictionary<Type, ConcreteTypeRepository> _shardedRepositories =
@@ -208,5 +242,50 @@ namespace ZeroLevel.Services.Collections
         {
             return this[typeof(T)].Keys();
         }
+
+        public void Save(string path)
+        {
+            using (var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None))
+            {
+                Save(stream);
+            }
+        }
+
+        public void Load(string path)
+        {
+            using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None))
+            {
+                Load(stream);
+            }
+        }
+
+        public void Save(Stream stream)
+        {
+            using (var writer = new MemoryStreamWriter(stream))
+            {
+                var keys = _shardedRepositories.Keys.ToArray();
+                writer.WriteInt32(keys.Length);
+                for (int i = 0; i < keys.Length; i++)
+                {
+                    writer.WriteString(keys[i].FullName);
+                    writer.Write(_shardedRepositories[keys[i]]);
+                }
+            }
+        }
+
+        public void Load(Stream stream)
+        {
+            using (var reader = new MemoryStreamReader(stream))
+            {
+                var count = reader.ReadInt32();
+                for (int i = 0; i < count; i++)
+                {
+                    var typeName = reader.ReadString();
+                    var type = Type.GetType(typeName);
+                    var rep = reader.Read<ConcreteTypeRepository>(type);
+                    _shardedRepositories.TryAdd(rep.GetEntityType(), rep);
+                }
+            }
+        }
     }
 }

ZeroLevel/Services/Collections/IEverythingStorage.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 
 namespace ZeroLevel.Services.Collections
 {
@@ -36,5 +37,13 @@ namespace ZeroLevel.Services.Collections
         void Remove(Type type, string key);
 
         object Get(Type type, string key);
+
+        void Save(string path);
+
+        void Load(string path);
+
+        void Save(Stream stream);
+
+        void Load(Stream stream);
     }
 }

ZeroLevel/Services/FileSystem/BigFileParser.cs

@@ -27,6 +27,7 @@ namespace ZeroLevel.Services.FileSystem
             _filePath = filePath;
             _parser = parser;
             _bufferSize = bufferSize;
+
         }
 
         public IEnumerable<IEnumerable<T>> ReadBatches(int batchSize)

ZeroLevel/Services/Logging/ILog.cs

@@ -1,52 +0,0 @@
-using System;
-
-namespace ZeroLevel.Services.Logging
-{
-    public interface ILog
-    {
-        /// <summary>
-        /// Message output as is, without adding a logging level and date
-        /// </summary>
-        void Raw(string line, params object[] args);
-
-        /// <summary>
-        /// Message
-        /// </summary>
-        void Info(string line, params object[] args);
-
-        /// <summary>
-        /// Warning
-        /// </summary>
-        void Warning(string line, params object[] args);
-
-        /// <summary>
-        /// Error
-        /// </summary>
-        void Error(string line, params object[] args);
-
-        /// <summary>
-        /// Error
-        /// </summary>
-        void Error(Exception ex, string line, params object[] args);
-
-        /// <summary>
-        /// Fatal crash
-        /// </summary>
-        void Fatal(string line, params object[] args);
-
-        /// <summary>
-        /// Fatal crash
-        /// </summary>
-        void Fatal(Exception ex, string line, params object[] args);
-
-        /// <summary>
-        /// Debug info
-        /// </summary>
-        void Debug(string line, params object[] args);
-
-        /// <summary>
-        /// Low Level Debug info
-        /// </summary>
-        void Verbose(string line, params object[] args);
-    }
-}

ZeroLevel/Services/Logging/Implementation/TextFileLogger.cs

@@ -6,7 +6,6 @@ using System.IO.Compression;
 using System.Linq;
 using System.Text;
 using ZeroLevel.Services.FileSystem;
-using ZeroLevel.Services.Logging;
 
 namespace ZeroLevel.Logging
 {

ZeroLevel/Services/Serialization/IBinaryReader.cs

@@ -95,6 +95,7 @@ namespace ZeroLevel.Services.Serialization
         #endregion
 
         T Read<T>() where T : IBinarySerializable;
+        T Read<T>(object arg) where T : IBinarySerializable;
 
         T ReadCompatible<T>();
 

ZeroLevel/Services/Serialization/MemoryStreamReader.cs

@@ -851,6 +851,15 @@ namespace ZeroLevel.Services.Serialization
             item.Deserialize(this);
             return item;
         }
+
+        public T Read<T>(object arg) where T : IBinarySerializable
+        {
+            byte type = ReadByte();
+            if (type == 0) return default(T);
+            var item = (T)Activator.CreateInstance(typeof(T), arg);
+            item.Deserialize(this);
+            return item;
+        }
         #endregion Extensions
 
         public void Dispose()

ZeroLevel/Services/Serialization/MemoryStreamWriter.cs

@@ -796,7 +796,6 @@ namespace ZeroLevel.Services.Serialization
                 }
             }
         }
-
         #endregion Extension
     }
 }