diff --git a/PartitionFileStorageTest/Program.cs b/PartitionFileStorageTest/Program.cs index f62a60a..c621acb 100644 --- a/PartitionFileStorageTest/Program.cs +++ b/PartitionFileStorageTest/Program.cs @@ -128,7 +128,7 @@ namespace PartitionFileStorageTest sw.Stop(); Log.Info($"Compress: {sw.ElapsedMilliseconds}ms"); sw.Restart(); - storePart.RebuildIndex(); + await storePart.RebuildIndex(); sw.Stop(); Log.Info($"Rebuild indexes: {sw.ElapsedMilliseconds}ms"); @@ -167,7 +167,7 @@ namespace PartitionFileStorageTest await readPart.RemoveKey(testKeys1[i], false); } sw.Restart(); - readPart.RebuildIndex(); + await readPart.RebuildIndex(); sw.Stop(); Log.Info($"Rebuild indexes after remove: {sw.ElapsedMilliseconds}ms"); Log.Info("Test #1 reading after remove"); @@ -296,7 +296,7 @@ namespace PartitionFileStorageTest sw.Stop(); Log.Info($"Compress: {sw.ElapsedMilliseconds}ms"); sw.Restart(); - storePart.RebuildIndex(); + await storePart.RebuildIndex(); } sw.Stop(); Log.Info($"Rebuild indexes: {sw.ElapsedMilliseconds}ms"); diff --git a/ZeroLevel/Services/PartitionStorage/Indexes/IndexBuilder.cs b/ZeroLevel/Services/PartitionStorage/Indexes/IndexBuilder.cs index b53ea3b..2bfbbd8 100644 --- a/ZeroLevel/Services/PartitionStorage/Indexes/IndexBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Indexes/IndexBuilder.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; +using ZeroLevel.Services.PartitionStorage.Interfaces; using ZeroLevel.Services.Serialization; namespace ZeroLevel.Services.PartitionStorage @@ -16,27 +18,30 @@ namespace ZeroLevel.Services.PartitionStorage private readonly string _indexCatalog; private readonly string _dataCatalog; private readonly int _stepValue; - private readonly Func _keyDeserializer; - private readonly Func _valueDeserializer; + private readonly IStoreKVSerializer Serializer; private readonly PhisicalFileAccessorCachee _phisicalFileAccessorCachee; - public IndexBuilder(IndexStepType indexType, int stepValue, string dataCatalog, PhisicalFileAccessorCachee phisicalFileAccessorCachee) + public IndexBuilder(IndexStepType indexType, + int stepValue, + string dataCatalog, + PhisicalFileAccessorCachee phisicalFileAccessorCachee, + IStoreKVSerializer serializer) { _dataCatalog = dataCatalog; _indexCatalog = Path.Combine(dataCatalog, INDEX_SUBFOLDER_NAME); _indexType = indexType; _stepValue = stepValue; - _keyDeserializer = MessageSerializer.GetDeserializer(); - _valueDeserializer = MessageSerializer.GetDeserializer(); + Serializer = serializer; _phisicalFileAccessorCachee = phisicalFileAccessorCachee; } /// /// Rebuild indexes for all files /// - internal void RebuildIndex() + internal async Task RebuildIndex() { var files = Directory.GetFiles(_dataCatalog); if (files != null && files.Length > 0) { + foreach (var file in files) { RebuildFileIndex(Path.GetFileName(file)); @@ -46,15 +51,15 @@ namespace ZeroLevel.Services.PartitionStorage /// /// Rebuild index for the specified file /// - internal void RebuildFileIndex(string file) + internal async Task RebuildFileIndex(string file) { if (_indexType == IndexStepType.AbsoluteCount) { - RebuildFileIndexWithAbsoluteCountIndexes(file); + await RebuildFileIndexWithAbsoluteCountIndexes(file); } else { - RebuildFileIndexWithSteps(file); + await RebuildFileIndexWithSteps(file); } } @@ -81,7 +86,7 @@ namespace ZeroLevel.Services.PartitionStorage /// /// Rebuild index with specified number of steps for specified file /// - private void RebuildFileIndexWithAbsoluteCountIndexes(string file) + private async Task RebuildFileIndexWithAbsoluteCountIndexes(string file) { if (false == Directory.Exists(_indexCatalog)) { @@ -93,9 +98,9 @@ namespace ZeroLevel.Services.PartitionStorage while (reader.EOS == false) { var pos = reader.Position; - var k = _keyDeserializer.Invoke(reader); - dict[k] = pos; - _valueDeserializer.Invoke(reader); + var k = await Serializer.KeyDeserializer.Invoke(reader); + dict[k.Value] = pos; + await Serializer.ValueDeserializer.Invoke(reader); } } if (dict.Count > _stepValue) @@ -130,7 +135,7 @@ namespace ZeroLevel.Services.PartitionStorage /// /// Rebuild index with specified step for keys /// - private void RebuildFileIndexWithSteps(string file) + private async Task RebuildFileIndexWithSteps(string file) { if (false == Directory.Exists(_indexCatalog)) { @@ -153,8 +158,8 @@ namespace ZeroLevel.Services.PartitionStorage { counter--; var pos = reader.Position; - var k = _keyDeserializer.Invoke(reader); - _valueDeserializer.Invoke(reader); + var k = await Serializer.KeyDeserializer.Invoke(reader); + await Serializer.ValueDeserializer.Invoke(reader); if (counter == 0) { writer.WriteCompatible(k); diff --git a/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionAccessor.cs b/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionAccessor.cs index 464a005..870aaff 100644 --- a/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionAccessor.cs +++ b/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionAccessor.cs @@ -16,7 +16,7 @@ namespace ZeroLevel.Services.PartitionStorage /// /// Rebuilds indexes for data in a partition /// - void RebuildIndex(); + Task RebuildIndex(); /// /// Search in a partition for a specified key /// diff --git a/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionBuilder.cs index cf0b162..2362f3e 100644 --- a/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionBuilder.cs @@ -32,6 +32,6 @@ namespace ZeroLevel.Services.PartitionStorage /// /// Rebuilds indexes for data in a partition /// - void RebuildIndex(); + Task RebuildIndex(); } } diff --git a/ZeroLevel/Services/PartitionStorage/Interfaces/IStoreSerializer.cs b/ZeroLevel/Services/PartitionStorage/Interfaces/IStoreSerializer.cs index 1b183af..7b1d821 100644 --- a/ZeroLevel/Services/PartitionStorage/Interfaces/IStoreSerializer.cs +++ b/ZeroLevel/Services/PartitionStorage/Interfaces/IStoreSerializer.cs @@ -4,18 +4,22 @@ using ZeroLevel.Services.Serialization; namespace ZeroLevel.Services.PartitionStorage.Interfaces { - public interface IStoreSerializer + public interface IStoreKVSerializer { Func KeySerializer { get; } - Func InputSerializer { get; } - Func ValueSerializer { get; } Func>> KeyDeserializer { get; } - Func>> InputDeserializer { get; } - Func>> ValueDeserializer { get; } } + + public interface IStoreSerializer + : IStoreKVSerializer + { + Func InputSerializer { get; } + + Func>> InputDeserializer { get; } + } } diff --git a/ZeroLevel/Services/PartitionStorage/Partition/BasePartition.cs b/ZeroLevel/Services/PartitionStorage/Partition/BasePartition.cs index ab31be5..4449c25 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/BasePartition.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/BasePartition.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Threading; +using System.Threading.Tasks; using ZeroLevel.Services.FileSystem; using ZeroLevel.Services.Memory; using ZeroLevel.Services.PartitionStorage.Interfaces; @@ -40,7 +41,7 @@ namespace ZeroLevel.Services.PartitionStorage.Partition Directory.CreateDirectory(_catalog); } _phisicalFileAccessor = fileAccessorCachee; - _indexBuilder = _options.Index.Enabled ? new IndexBuilder(_options.Index.StepType, _options.Index.StepValue, _catalog, fileAccessorCachee) : null; + _indexBuilder = _options.Index.Enabled ? new IndexBuilder(_options.Index.StepType, _options.Index.StepValue, _catalog, fileAccessorCachee, Serializer) : null; Serializer = serializer; } @@ -60,21 +61,21 @@ namespace ZeroLevel.Services.PartitionStorage.Partition /// /// Rebuild indexes for all files /// - protected void RebuildIndexes() + protected async Task RebuildIndexes() { if (_options.Index.Enabled) { - _indexBuilder.RebuildIndex(); + await _indexBuilder.RebuildIndex(); } } /// /// Rebuild index for the specified file /// - internal void RebuildFileIndex(string file) + internal async Task RebuildFileIndex(string file) { if (_options.Index.Enabled) { - _indexBuilder.RebuildFileIndex(file); + await _indexBuilder.RebuildFileIndex(file); } } /// diff --git a/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs index cbe32ef..ddd8046 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs @@ -84,7 +84,7 @@ namespace ZeroLevel.Services.PartitionStorage.Partition } } } - public void RebuildIndex() => RebuildIndexes(); + public async Task RebuildIndex() => await RebuildIndexes(); #endregion #region Private methods diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs b/ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs index 356869f..a39769b 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs @@ -132,7 +132,7 @@ namespace ZeroLevel.Services.PartitionStorage } // 3. Rebuil index - (_accessor as BasePartition).RebuildFileIndex(name); + await (_accessor as BasePartition).RebuildFileIndex(name); } } // remove temporary files diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs index e250b03..94841b1 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs @@ -159,9 +159,9 @@ namespace ZeroLevel.Services.PartitionStorage } } } - public void RebuildIndex() + public async Task RebuildIndex() { - RebuildIndexes(); + await RebuildIndexes(); if (_options.Index.Enabled) { Indexes.ResetCachee(); @@ -442,7 +442,7 @@ namespace ZeroLevel.Services.PartitionStorage // Rebuild index if needs if (_options.Index.Enabled && autoReindex) { - RebuildFileIndex(filePath); + await RebuildFileIndex(filePath); } } } diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs index 930a2fe..b36be88 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs @@ -88,7 +88,7 @@ namespace ZeroLevel.Services.PartitionStorage } } } - public void RebuildIndex() => RebuildIndexes(); + public async Task RebuildIndex() => await RebuildIndexes(); #endregion #region Private methods diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index dda8ef0..dfdc311 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -6,7 +6,7 @@ ogoun ogoun - 3.4.0.1 + 3.4.0.2 KVDB ia async now https://github.com/ogoun/Zero/wiki Copyright Ogoun 2023 @@ -14,8 +14,8 @@ https://github.com/ogoun/Zero git - 3.4.0.1 - 3.4.0.1 + 3.4.0.2 + 3.4.0.2 AnyCPU;x64;x86 zero.png full