Update

1. Configuration
- Fix ini reader
- New methods for merge configs
- New method GetOrCreateSection in ConfigurationSet
- New reader for read config from Environment Variables
- Refactoring
2. Partition storage
- Refactoring

PartitionFileStorageTest/Program.cs

@@ -17,7 +17,7 @@ namespace PartitionFileStorageTest
     internal class Program
     {
         //        const int PAIRS_COUNT = 200_000_000;
-        const long PAIRS_COUNT = 100_000_000;
+        const long PAIRS_COUNT = 10_000_000;
 
         private class Metadata
         {
@@ -298,7 +298,8 @@ namespace PartitionFileStorageTest
 
             ulong totalData = 0;
             ulong totalKeys = 0;
-            var readPart = store.CreateAccessor(meta);
+            using (var readPart = store.CreateAccessor(meta))
+            {
                 /*
                 Log.Info("Test #1 reading");
                 foreach (var key in testKeys1)
@@ -363,6 +364,7 @@ namespace PartitionFileStorageTest
                     totalData += (ulong)(e.Value?.Length ?? 0);
                     totalKeys++;
                 }
+            }
             Log.Info($"\t\tFound: {totalKeys}/{Keys.Count} keys. {totalData} bytes");
             store.Dispose();
             Log.Info("Completed");
@@ -487,7 +489,7 @@ namespace PartitionFileStorageTest
 
         static void Main(string[] args)
         {
-            FaultCompressionTest(@"F:\Desktop\DATKA\DNS", new StoreMetadata { Date = new DateTime(2023, 01, 20) });
+            //FaultCompressionTest(@"F:\Desktop\DATKA\DNS", new StoreMetadata { Date = new DateTime(2023, 01, 20) });
 
             var root = @"H:\temp";
             var options = new StoreOptions<ulong, ulong, byte[], Metadata>

ZeroLevel.HNSW/ZeroLevel.HNSW.csproj

@@ -5,7 +5,7 @@
     <Platforms>AnyCPU;x64</Platforms>
     <PlatformTarget>x64</PlatformTarget>
     <DebugType>full</DebugType>
-    <Version>1.0.0.4</Version>
+    <Version>1.0.0.5</Version>
     <Company>ogoun</Company>
     <Authors>Ogoun</Authors>
     <Copyright>Copyright Ogoun 2022</Copyright>

ZeroLevel.UnitTests/ConfigurationTest.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using Xunit;
 
 namespace ZeroLevel.UnitTests
@@ -95,5 +96,52 @@ namespace ZeroLevel.UnitTests
             Assert.Equal(f, set.Default.First<float>("f"));
             Assert.Equal(i, set.Default.First<int>("i"));
         }
+
+
+        [Fact]
+        public void MergeTest()
+        {
+            // ARRANGE
+            var set1 = Configuration.CreateSet();
+            set1.Default.Append("dk1", "1");
+            set1.Default.Append("dk1", "1");
+            set1["A"].Append("Ak1", "ak1");
+            set1["A"].Append("Ak2", "ak2");
+            var set2 = Configuration.CreateSet();
+            set2.Default.Append("dk1", "2");
+            set2["A"].Append("Ak1", "ak1");
+            var set3 = Configuration.CreateSet();
+            set3.Default.Append("dk1", "3");
+            set3["A"].Append("Ak1", "ak2");
+
+            // ACT 1
+            var mergedSet1 = Configuration.Merge(Services.Config.ConfigurationRecordExistBehavior.Append, set1, set2, set3);
+            // ASSERT 1
+            Assert.Equal(mergedSet1.Default["dk1"].Count(), 4);
+            Assert.Contains(mergedSet1.Default["dk1"], i => i == "1");
+            Assert.Contains(mergedSet1.Default["dk1"], i => i == "2");
+            Assert.Contains(mergedSet1.Default["dk1"], i => i == "3");
+
+            Assert.Equal(mergedSet1["A"]["Ak1"].Count(), 3);
+            Assert.Equal(mergedSet1["A"]["Ak2"].Count(), 1);
+
+            // ACT 2
+            var mergedSet2 = Configuration.Merge(Services.Config.ConfigurationRecordExistBehavior.IgnoreNew, set1, set2, set3);
+            // ASSERT 2
+            Assert.Equal(mergedSet2.Default["dk1"].Count(), 2);
+            Assert.Contains(mergedSet2.Default["dk1"], i => i == "1");
+
+            Assert.Equal(mergedSet2["A"]["Ak1"].Count(), 1);
+            Assert.Equal(mergedSet2["A"]["Ak2"].Count(), 1);
+
+            // ACT 3
+            var mergedSet3 = Configuration.Merge(Services.Config.ConfigurationRecordExistBehavior.Overwrite, set1, set2, set3);
+            // ASSERT 3
+            Assert.Equal(mergedSet3.Default["dk1"].Count(), 1);
+            Assert.Contains(mergedSet3.Default["dk1"], i => i == "3");
+
+            Assert.Equal(mergedSet3["A"]["Ak1"].Count(), 1);
+            Assert.Equal(mergedSet3["A"]["Ak2"].Count(), 1);
+        }
     }
 }

ZeroLevel/Services/Config/BaseConfiguration.cs

@@ -6,30 +6,12 @@ using System.Linq;
 using System.Net;
 using System.Reflection;
 using ZeroLevel.Services.Collections;
-using ZeroLevel.Services.Invokation;
 using ZeroLevel.Services.ObjectMapping;
 using ZeroLevel.Services.Reflection;
 using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel.Services.Config
 {
-    public interface IConfigRecordParser
-    {
-        object Parse(string line);
-    }
-
-    public class ConfigRecordParseAttribute
-        : Attribute
-    {
-        public IConfigRecordParser Parser { get; set; }
-
-        public ConfigRecordParseAttribute(Type parserType)
-        {
-            if (parserType == null) throw new ArgumentNullException(nameof(parserType));
-            Parser = (IConfigRecordParser)Activator.CreateInstance(parserType);
-        }
-    }
-
     /// <summary>
     /// Base configuration
     /// </summary>
@@ -489,6 +471,32 @@ namespace ZeroLevel.Services.Config
             }
         }
 
+        public void MergeFrom(IConfiguration config, ConfigurationRecordExistBehavior existRecordBehavior)
+        {
+            foreach (var key in config.Keys)
+            {
+                if (this.Contains(key))
+                {
+                    switch (existRecordBehavior)
+                    {
+                        case ConfigurationRecordExistBehavior.Append:
+                            this.Append(key, config[key]);
+                            break;
+                        case ConfigurationRecordExistBehavior.IgnoreNew:
+                            continue;
+                        case ConfigurationRecordExistBehavior.Overwrite:
+                            this.Remove(key);
+                            this.Append(key, config[key]);
+                            break;
+                    }
+                }
+                else
+                {
+                    this.Append(key, config[key]);
+                }
+            }
+        }
+
         public T Bind<T>() => (T)Bind(typeof(T));
         public object Bind(Type type)
         {

ZeroLevel/Services/Config/BaseConfigurationSet.cs

@@ -3,7 +3,6 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Globalization;
 using ZeroLevel.Services.ObjectMapping;
-using ZeroLevel.Services.Reflection;
 using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel.Services.Config
@@ -122,6 +121,17 @@ namespace ZeroLevel.Services.Config
             throw new KeyNotFoundException("Section not found: " + sectionName);
         }
 
+        public IConfiguration GetOrCreateSection(string sectionName)
+        {
+            var key = GetKey(sectionName);
+            IConfiguration exists;
+            if (_sections.TryGetValue(key, out exists))
+            {
+                return exists;
+            }
+            _sections.TryAdd(key, new BaseConfiguration());
+            return _sections[key];
+        }
         public bool ContainsSection(string sectionName)
         {
             var key = GetKey(sectionName);

ZeroLevel/Services/Config/ConfigRecordParseAttribute.cs

@@ -0,0 +1,16 @@
+using System;
+
+namespace ZeroLevel.Services.Config
+{
+    public class ConfigRecordParseAttribute
+        : Attribute
+    {
+        public IConfigRecordParser Parser { get; set; }
+
+        public ConfigRecordParseAttribute(Type parserType)
+        {
+            if (parserType == null) throw new ArgumentNullException(nameof(parserType));
+            Parser = (IConfigRecordParser)Activator.CreateInstance(parserType);
+        }
+    }
+}

ZeroLevel/Services/Config/Configuration.cs

@@ -131,6 +131,19 @@ namespace ZeroLevel
         #endregion Factory
 
         #region Read configuration
+        public static IConfiguration ReadFromEnvironmentVariables()
+        {
+            try
+            {
+                return new EnvironmentVariablesConfigReader().ReadConfiguration();
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex, $"[Configuration.ReadFromEnvironmentVariables] Can't read environment variables");
+                throw;
+            }
+        }
+
         /// <summary>
         /// Creating a configuration from the AppSettings section of the app.config or web.config file
         /// </summary>
@@ -388,6 +401,32 @@ namespace ZeroLevel
         }
         #endregion Read configuration
 
+
+
+        public static IConfiguration Merge(ConfigurationRecordExistBehavior existRecordBehavior, params IConfiguration[] configurations)
+        {
+            var result = Configuration.Create();
+            foreach (var configuration in configurations)
+            {
+                result.MergeFrom(configuration, existRecordBehavior);
+            }
+            return result;
+        }
+
+        public static IConfigurationSet Merge(ConfigurationRecordExistBehavior existRecordBehavior, params IConfigurationSet[] configurationSets)
+        {
+            var result = Configuration.CreateSet();
+            foreach (var set in configurationSets)
+            {
+                foreach (var sectionName in set.SectionNames)
+                {
+                    var section = result.GetOrCreateSection(sectionName);
+                    section.MergeFrom(set[sectionName], existRecordBehavior);
+                }
+            }
+            return result;
+        }
+
         #region Write configuration
 
         /// <summary>

ZeroLevel/Services/Config/ConfigurationRecordExistBehavior.cs

@@ -0,0 +1,22 @@
+namespace ZeroLevel.Services.Config
+{
+    /// <summary>
+    /// Merge behavior when keys match
+    /// </summary>
+    public enum ConfigurationRecordExistBehavior
+        : int
+    {
+        /// <summary>
+        /// Add values to existing values
+        /// </summary>
+        Append = 0,
+        /// <summary>
+        /// Overwrite existing values
+        /// </summary>
+        Overwrite = 1,
+        /// <summary>
+        /// Ignore new values
+        /// </summary>
+        IgnoreNew = 2
+    }
+}

ZeroLevel/Services/Config/IConfigRecordParser.cs

@@ -0,0 +1,7 @@
+namespace ZeroLevel.Services.Config
+{
+    public interface IConfigRecordParser
+    {
+        object Parse(string line);
+    }
+}

ZeroLevel/Services/Config/IConfiguration.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using ZeroLevel.Services.Config;
 using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel
@@ -141,6 +142,7 @@ namespace ZeroLevel
         #endregion Create, Clean, Delete
 
         void CopyTo(IConfiguration config);
+        void MergeFrom(IConfiguration config, ConfigurationRecordExistBehavior existRecordBehavior);
 
         T Bind<T>();
         object Bind(Type type);

ZeroLevel/Services/Config/IConfigurationSet.cs

@@ -57,6 +57,8 @@ namespace ZeroLevel
         /// <returns>Data section</returns>
         IConfiguration GetSection(string sectionName);
 
+        IConfiguration GetOrCreateSection(string sectionName);
+
         /// <summary>
         /// Check for a section by name
         /// </summary>

ZeroLevel/Services/Config/Implementation/EnvironmentVariablesConfigReader.cs

@@ -0,0 +1,26 @@
+using System;
+
+namespace ZeroLevel.Services.Config.Implementation
+{
+    internal sealed class EnvironmentVariablesConfigReader
+        : IConfigurationReader
+    {
+        public IConfiguration ReadConfiguration()
+        {
+            var result = Configuration.Create();
+            var enumerator = Environment.GetEnvironmentVariables().GetEnumerator();
+            while (enumerator.MoveNext())
+            {
+                string key = (string)enumerator.Entry.Key;
+                string value = ((string?)enumerator.Entry.Value) ?? string.Empty;
+                result.Append(key, value);
+            }
+            return result;
+        }
+
+        public IConfigurationSet ReadConfigurationSet()
+        {
+            return Configuration.CreateSet(ReadConfiguration());
+        }
+    }
+}

ZeroLevel/Services/Config/Implementation/IniFileReader.cs

@@ -53,8 +53,13 @@ namespace ZeroLevel.Services.Config.Implementation
                     key = originalKey.ToLower(CultureInfo.InvariantCulture);
                     value = string.Empty;
                 }
-                if (key[0].Equals(';') || key[0].Equals('#'))
+                if (key[0].Equals(';') || key[0].Equals('#') || key[0].Equals('/'))
                     continue;
+                // Remove quotes
+                if (value.Length > 1 && value[0] == '"' && value[value.Length - 1] == '"')
+                {
+                    value = value.Substring(1, value.Length - 2);
+                }
                 if (string.IsNullOrEmpty(value) && key[0].Equals('[') && key[key.Length - 1].Equals(']'))
                 {
                     sectionName = originalKey.Trim('[', ']');
@@ -100,8 +105,13 @@ namespace ZeroLevel.Services.Config.Implementation
                     key = originalKey.ToLower(CultureInfo.InvariantCulture);
                     value = string.Empty;
                 }
-                if (key[0].Equals(';') || key[0].Equals('#'))
+                if (key[0].Equals(';') || key[0].Equals('#') || key[0].Equals('/'))
                     continue;
+                // Remove quotes
+                if (value.Length > 1 && value[0] == '"' && value[value.Length - 1] == '"')
+                {
+                    value = value.Substring(1, value.Length - 2);
+                }
                 if (string.IsNullOrEmpty(value) && key[0].Equals('[') && key[key.Length - 1].Equals(']'))
                 {
                     sectionName = originalKey.Trim('[', ']');

ZeroLevel/Services/PartitionStorage/Indexes/KeysIndex/StorePartitionSparseIndex.cs

@@ -111,7 +111,13 @@ namespace ZeroLevel.Services.PartitionStorage
         {
             if (_enableIndexInMemoryCachee)
             {
-                _phisicalFileAccessorCachee.DropAllIndexReaders();
+                if (Directory.Exists(_indexFolder))
+                {
+                    foreach (var file in Directory.GetFiles(_indexFolder))
+                    {
+                        _phisicalFileAccessorCachee.DropIndexReader(file);
+                    }
+                }
             }
         }
 

ZeroLevel/Services/PartitionStorage/Partition/BasePartition.cs

@@ -51,9 +51,12 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
         public void Dispose()
         {
             CloseWriteStreams();
+            Release();            
         }
         #endregion
 
+        public abstract void Release();
+
         /// <summary>
         /// Rebuild indexes for all files
         /// </summary>

ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs

@@ -169,6 +169,17 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
                 PhisicalFileAccessorCachee.UnlockFile(file);
             }
         }
+
+        public override void Release()
+        {
+            if (Directory.Exists(_catalog))
+            {
+                foreach (var file in Directory.GetFiles(_catalog))
+                {
+                    PhisicalFileAccessorCachee.DropDataReader(file);
+                }
+            }
+        }
         #endregion
     }
 }

ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs

@@ -477,5 +477,17 @@ namespace ZeroLevel.Services.PartitionStorage
             target.Write(buffer, 0, buffer.Length);
         }
         #endregion
+
+        public override void Release()
+        {
+            if (Directory.Exists(_catalog))
+            {
+                foreach (var file in Directory.GetFiles(_catalog))
+                {
+                    PhisicalFileAccessorCachee.DropDataReader(file);
+                }
+            }
+            Indexes.ResetCachee();
+        }
     }
 }

ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs

@@ -5,7 +5,6 @@ using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using ZeroLevel.Services.FileSystem;
-using ZeroLevel.Services.Memory;
 using ZeroLevel.Services.PartitionStorage.Interfaces;
 using ZeroLevel.Services.PartitionStorage.Partition;
 using ZeroLevel.Services.Serialization;
@@ -199,6 +198,10 @@ namespace ZeroLevel.Services.PartitionStorage
                 PhisicalFileAccessorCachee.UnlockFile(file);
             }
         }
+
+        public override void Release()
+        {
+        }
         #endregion
     }
 }

ZeroLevel/ZeroLevel.csproj

@@ -6,16 +6,16 @@
 </Description>
     <Authors>ogoun</Authors>
     <Company>ogoun</Company>
-    <AssemblyVersion>3.3.9.3</AssemblyVersion>
+    <AssemblyVersion>3.3.9.5</AssemblyVersion>
-    <PackageReleaseNotes>Update Bootstrap</PackageReleaseNotes>
+    <PackageReleaseNotes>Partition storage. Fix.</PackageReleaseNotes>
     <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
     <Copyright>Copyright Ogoun 2023</Copyright>
     <PackageLicenseUrl></PackageLicenseUrl>
     <PackageIconUrl></PackageIconUrl>
     <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
-    <Version>3.3.9.3</Version>
+    <Version>3.3.9.5</Version>
-    <FileVersion>3.3.9.3</FileVersion>
+    <FileVersion>3.3.9.5</FileVersion>
     <Platforms>AnyCPU;x64;x86</Platforms>
     <PackageIcon>zero.png</PackageIcon>
     <DebugType>full</DebugType>