diff --git a/PartitionFileStorageTest/Program.cs b/PartitionFileStorageTest/Program.cs index 8bd0fa7..467ce4a 100644 --- a/PartitionFileStorageTest/Program.cs +++ b/PartitionFileStorageTest/Program.cs @@ -3,12 +3,17 @@ using System.Diagnostics; using ZeroLevel; using ZeroLevel.Collections; using ZeroLevel.Services.FileSystem; +using ZeroLevel.Services.HashFunctions; using ZeroLevel.Services.Memory; using ZeroLevel.Services.PartitionStorage; using ZeroLevel.Services.Serialization; namespace PartitionFileStorageTest { + public class StoreMetadata + { + public DateTime Date { get; set; } + } internal class Program { // const int PAIRS_COUNT = 200_000_000; @@ -430,7 +435,7 @@ namespace PartitionFileStorageTest { try { - serializer.KeyDeserializer.Invoke(reader, out var key); + serializer.KeyDeserializer.Invoke(reader, out var key); if (false == dict.ContainsKey(key)) { dict[key] = new HashSet(); @@ -450,8 +455,40 @@ namespace PartitionFileStorageTest } } + private static void FaultCompressionTest(string folder, StoreMetadata meta) + { + var options = new StoreOptions + { + Index = new IndexOptions + { + Enabled = true, + StepType = IndexStepType.Step, + StepValue = 32, + EnableIndexInMemoryCachee = true + }, + RootFolder = folder, + FilePartition = new StoreFilePartition("Host hash", (key, _) => Math.Abs(StringHash.DotNetFullHash(key) % 367).ToString()), + MergeFunction = list => + { + ulong s = 0; + return Compressor.GetEncodedBytes(list.OrderBy(c => c), ref s); + }, + Partitions = new List> + { + new StoreCatalogPartition("Date", m => m.Date.ToString("yyyyMMdd")), + }, + KeyComparer = (left, right) => string.Compare(left, right, true), + ThreadSafeWriting = true + }; + var store = new Store(options); + var builder = store.CreateBuilder(meta); + builder.Compress(); + } + static void Main(string[] args) { + FaultCompressionTest(@"F:\Desktop\DATKA\DNS", new StoreMetadata { Date = new DateTime(2023, 01, 20) }); + var root = @"H:\temp"; var options = new StoreOptions { diff --git a/ZeroLevel/Services/PartitionStorage/Interfaces/IStore.cs b/ZeroLevel/Services/PartitionStorage/Interfaces/IStore.cs index 34f9538..d81dbb8 100644 --- a/ZeroLevel/Services/PartitionStorage/Interfaces/IStore.cs +++ b/ZeroLevel/Services/PartitionStorage/Interfaces/IStore.cs @@ -34,5 +34,9 @@ namespace ZeroLevel.Services.PartitionStorage /// Deleting a partition /// void RemovePartition(TMeta info); + /// + /// Remove all cached data accessors + /// + void DropCache(); } } diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs index eb15b6e..83129e5 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs @@ -144,7 +144,7 @@ namespace ZeroLevel.Services.PartitionStorage try { var dict = new Dictionary>(); - var accessor = new StreamVewAccessor(new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.None, 1024 * 1024 * 32)); + var accessor = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.None, 1024 * 1024 * 32); if (accessor != null) { using (var reader = new MemoryStreamReader(accessor)) @@ -155,19 +155,26 @@ namespace ZeroLevel.Services.PartitionStorage { throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read key."); } - if (false == dict.ContainsKey(key)) + if (key != null) { - dict[key] = new HashSet(); + if (false == dict.ContainsKey(key)) + { + dict[key] = new HashSet(); + } + if (reader.EOS) + { + break; + } + if (Serializer.InputDeserializer.Invoke(reader, out input) == false) + { + throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read input value."); + } + dict[key].Add(input); } - if (reader.EOS) + else { - break; + Log.SystemWarning($"[StorePartitionBuilder.CompressFile] Null-value key in file '{file}'"); } - if (Serializer.InputDeserializer.Invoke(reader, out input) == false) - { - throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read input value."); - } - dict[key].Add(input); } } } diff --git a/ZeroLevel/Services/PartitionStorage/PhisicalFileAccessorCachee.cs b/ZeroLevel/Services/PartitionStorage/PhisicalFileAccessorCachee.cs index 6b277ad..b69c3c6 100644 --- a/ZeroLevel/Services/PartitionStorage/PhisicalFileAccessorCachee.cs +++ b/ZeroLevel/Services/PartitionStorage/PhisicalFileAccessorCachee.cs @@ -1,6 +1,6 @@ using System; -using System.Collections.Generic; using System.IO; +using ZeroLevel.Collections; using ZeroLevel.Services.Cache; using ZeroLevel.Services.FileSystem; using ZeroLevel.Services.Memory; @@ -13,7 +13,7 @@ namespace ZeroLevel.Services.PartitionStorage private readonly TimerCachee _indexReadersCachee; private readonly TimerCachee _dataReadersCachee; - private readonly HashSet _lockedFiles = new HashSet(); + private readonly ConcurrentHashSet _lockedFiles = new ConcurrentHashSet(); public PhisicalFileAccessorCachee(TimeSpan dataExpirationPeriod, TimeSpan indexExpirationPeriod) { @@ -140,11 +140,12 @@ namespace ZeroLevel.Services.PartitionStorage public void UnlockFile(string filePath) { - _lockedFiles.Remove(filePath); + _lockedFiles.TryRemove(filePath); } public void Dispose() { + _lockedFiles.Clear(); _dataReadersCachee.Dispose(); _indexReadersCachee.Dispose(); } diff --git a/ZeroLevel/Services/PartitionStorage/Store.cs b/ZeroLevel/Services/PartitionStorage/Store.cs index 2b7d2e8..20e240b 100644 --- a/ZeroLevel/Services/PartitionStorage/Store.cs +++ b/ZeroLevel/Services/PartitionStorage/Store.cs @@ -58,6 +58,12 @@ namespace ZeroLevel.Services.PartitionStorage return new StoreMergePartitionAccessor(_options, info, decompressor, _serializer, _fileAccessorCachee); } + public void DropCache() + { + _fileAccessorCachee.DropAllDataReaders(); + _fileAccessorCachee.DropAllIndexReaders(); + } + public async Task> Search(StoreSearchRequest searchRequest) { var result = new StoreSearchResult(); diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index 2717624..68130df 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -6,16 +6,16 @@ ogoun ogoun - 3.3.9.0 - Partition storage. Fix concurrent work + 3.3.9.2 + Partition storage. DropCache method https://github.com/ogoun/Zero/wiki Copyright Ogoun 2023 https://github.com/ogoun/Zero git - 3.3.9.0 - 3.3.9.0 + 3.3.9.2 + 3.3.9.2 AnyCPU;x64;x86 zero.png full