PartitionStorage DropCache

PartitionFileStorageTest/Program.cs

@@ -3,12 +3,17 @@ using System.Diagnostics;
 using ZeroLevel;
 using ZeroLevel.Collections;
 using ZeroLevel.Services.FileSystem;
+using ZeroLevel.Services.HashFunctions;
 using ZeroLevel.Services.Memory;
 using ZeroLevel.Services.PartitionStorage;
 using ZeroLevel.Services.Serialization;
 
 namespace PartitionFileStorageTest
 {
+    public class StoreMetadata
+    {
+        public DateTime Date { get; set; }
+    }
     internal class Program
     {
         //        const int PAIRS_COUNT = 200_000_000;
@@ -430,7 +435,7 @@ namespace PartitionFileStorageTest
                 {
                     try
                     {
-                         serializer.KeyDeserializer.Invoke(reader, out var  key);
+                        serializer.KeyDeserializer.Invoke(reader, out var key);
                         if (false == dict.ContainsKey(key))
                         {
                             dict[key] = new HashSet<ulong>();
@@ -450,8 +455,40 @@ namespace PartitionFileStorageTest
             }
         }
 
+        private static void FaultCompressionTest(string folder, StoreMetadata meta)
+        {
+            var options = new StoreOptions<string, ulong, byte[], StoreMetadata>
+            {
+                Index = new IndexOptions
+                {
+                    Enabled = true,
+                    StepType = IndexStepType.Step,
+                    StepValue = 32,
+                    EnableIndexInMemoryCachee = true
+                },
+                RootFolder = folder,
+                FilePartition = new StoreFilePartition<string, StoreMetadata>("Host hash", (key, _) => Math.Abs(StringHash.DotNetFullHash(key) % 367).ToString()),
+                MergeFunction = list =>
+                {
+                    ulong s = 0;
+                    return Compressor.GetEncodedBytes(list.OrderBy(c => c), ref s);
+                },
+                Partitions = new List<StoreCatalogPartition<StoreMetadata>>
+                {
+                    new StoreCatalogPartition<StoreMetadata>("Date", m => m.Date.ToString("yyyyMMdd")),
+                },
+                KeyComparer = (left, right) => string.Compare(left, right, true),
+                ThreadSafeWriting = true
+            };
+            var store = new Store<string, ulong, byte[], StoreMetadata>(options);
+            var builder = store.CreateBuilder(meta);
+            builder.Compress();
+        }
+
         static void Main(string[] args)
         {
+            FaultCompressionTest(@"F:\Desktop\DATKA\DNS", new StoreMetadata { Date = new DateTime(2023, 01, 20) });
+
             var root = @"H:\temp";
             var options = new StoreOptions<ulong, ulong, byte[], Metadata>
             {

ZeroLevel/Services/PartitionStorage/Interfaces/IStore.cs

@@ -34,5 +34,9 @@ namespace ZeroLevel.Services.PartitionStorage
         /// Deleting a partition
         /// </summary>
         void RemovePartition(TMeta info);
+        /// <summary>
+        /// Remove all cached data accessors
+        /// </summary>
+        void DropCache();
     }
 }

ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs

@@ -144,7 +144,7 @@ namespace ZeroLevel.Services.PartitionStorage
             try
             {
                 var dict = new Dictionary<TKey, HashSet<TInput>>();
-                var accessor = new StreamVewAccessor(new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.None, 1024 * 1024 * 32));
+                var accessor = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.None, 1024 * 1024 * 32);
                 if (accessor != null)
                 {
                     using (var reader = new MemoryStreamReader(accessor))
@@ -155,19 +155,26 @@ namespace ZeroLevel.Services.PartitionStorage
                             {
                                 throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read key.");
                             }
-                            if (false == dict.ContainsKey(key))
+                            if (key != null)
                             {
-                                dict[key] = new HashSet<TInput>();
+                                if (false == dict.ContainsKey(key))
+                                {
+                                    dict[key] = new HashSet<TInput>();
+                                }
+                                if (reader.EOS)
+                                {
+                                    break;
+                                }
+                                if (Serializer.InputDeserializer.Invoke(reader, out input) == false)
+                                {
+                                    throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read input value.");
+                                }
+                                dict[key].Add(input);
                             }
-                            if (reader.EOS)
+                            else
                             {
-                                break;
+                                Log.SystemWarning($"[StorePartitionBuilder.CompressFile] Null-value key in file '{file}'");
                             }
-                            if (Serializer.InputDeserializer.Invoke(reader, out input) == false)
-                            {
-                                throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read input value.");
-                            }
-                            dict[key].Add(input);
                         }
                     }
                 }

ZeroLevel/Services/PartitionStorage/PhisicalFileAccessorCachee.cs

@@ -1,6 +1,6 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
+using ZeroLevel.Collections;
 using ZeroLevel.Services.Cache;
 using ZeroLevel.Services.FileSystem;
 using ZeroLevel.Services.Memory;
@@ -13,7 +13,7 @@ namespace ZeroLevel.Services.PartitionStorage
         private readonly TimerCachee<ParallelFileReader> _indexReadersCachee;
         private readonly TimerCachee<ParallelFileReader> _dataReadersCachee;
 
-        private readonly HashSet<string> _lockedFiles = new HashSet<string>();
+        private readonly ConcurrentHashSet<string> _lockedFiles = new ConcurrentHashSet<string>();
 
         public PhisicalFileAccessorCachee(TimeSpan dataExpirationPeriod, TimeSpan indexExpirationPeriod)
         {
@@ -140,11 +140,12 @@ namespace ZeroLevel.Services.PartitionStorage
 
         public void UnlockFile(string filePath) 
         {
-            _lockedFiles.Remove(filePath);
+            _lockedFiles.TryRemove(filePath);
         }
 
         public void Dispose()
         {
+            _lockedFiles.Clear();
             _dataReadersCachee.Dispose();
             _indexReadersCachee.Dispose();
         }

ZeroLevel/Services/PartitionStorage/Store.cs

@@ -58,6 +58,12 @@ namespace ZeroLevel.Services.PartitionStorage
             return new StoreMergePartitionAccessor<TKey, TInput, TValue, TMeta>(_options, info, decompressor, _serializer, _fileAccessorCachee);
         }
 
+        public void DropCache()
+        {
+            _fileAccessorCachee.DropAllDataReaders();
+            _fileAccessorCachee.DropAllIndexReaders();
+        }
+
         public async Task<StoreSearchResult<TKey, TValue, TMeta>> Search(StoreSearchRequest<TKey, TMeta> searchRequest)
         {
             var result = new StoreSearchResult<TKey, TValue, TMeta>();

ZeroLevel/ZeroLevel.csproj

@@ -6,16 +6,16 @@
 </Description>
     <Authors>ogoun</Authors>
     <Company>ogoun</Company>
-    <AssemblyVersion>3.3.9.0</AssemblyVersion>
-    <PackageReleaseNotes>Partition storage. Fix concurrent work</PackageReleaseNotes>
+    <AssemblyVersion>3.3.9.2</AssemblyVersion>
+    <PackageReleaseNotes>Partition storage. DropCache method</PackageReleaseNotes>
     <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
     <Copyright>Copyright Ogoun 2023</Copyright>
     <PackageLicenseUrl></PackageLicenseUrl>
     <PackageIconUrl></PackageIconUrl>
     <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
-    <Version>3.3.9.0</Version>
-    <FileVersion>3.3.9.0</FileVersion>
+    <Version>3.3.9.2</Version>
+    <FileVersion>3.3.9.2</FileVersion>
     <Platforms>AnyCPU;x64;x86</Platforms>
     <PackageIcon>zero.png</PackageIcon>
     <DebugType>full</DebugType>