diff --git a/PartitionFileStorageTest/Program.cs b/PartitionFileStorageTest/Program.cs index 1d953bf..6b8f88f 100644 --- a/PartitionFileStorageTest/Program.cs +++ b/PartitionFileStorageTest/Program.cs @@ -9,7 +9,8 @@ namespace PartitionFileStorageTest { internal class Program { - const int PAIRS_COUNT = 200_000_000; +// const int PAIRS_COUNT = 200_000_000; + const int PAIRS_COUNT = 200_000; private class Metadata { @@ -75,7 +76,6 @@ namespace PartitionFileStorageTest var testKeys1 = new List(); var testKeys2 = new List(); var testData = new Dictionary>(); - var total = 0L; var insertCount = (int)(0.7 * pairs.Count); @@ -86,7 +86,6 @@ namespace PartitionFileStorageTest if (testData.ContainsKey(key) == false) testData[key] = new HashSet(); testData[key].Add(val); storePart.Store(key, val); - total++; if (key % 717 == 0) { testKeys1.Add(key); @@ -98,7 +97,7 @@ namespace PartitionFileStorageTest } sw.Stop(); - Log.Info($"Fill journal: {sw.ElapsedMilliseconds}ms"); + Log.Info($"Fill journal: {sw.ElapsedMilliseconds}ms. Records writed: {storePart.TotalRecords}"); sw.Restart(); storePart.CompleteAdding(); storePart.Compress(); @@ -116,12 +115,11 @@ namespace PartitionFileStorageTest { var key = pairs[i].Item1; var val = pairs[i].Item2; - total++; if (testData.ContainsKey(key) == false) testData[key] = new HashSet(); testData[key].Add(val); merger.Store(key, val); } - Log.Info($"Merge journal filled: {sw.ElapsedMilliseconds}ms. Total data count: {total}"); + Log.Info($"Merge journal filled: {sw.ElapsedMilliseconds}ms. New records merged: {merger.TotalRecords}"); merger.Compress(); // auto reindex sw.Stop(); Log.Info($"Compress after merge: {sw.ElapsedMilliseconds}ms"); @@ -396,8 +394,10 @@ namespace PartitionFileStorageTest // FastTest(options); FSUtils.CleanAndTestFolder(root); FullStoreTest(options, pairs); + /* FSUtils.CleanAndTestFolder(root); FullStoreMultithreadTest(optionsMultiThread, pairs); + */ Console.ReadKey(); } } diff --git a/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionBuilder.cs index 2310a23..2a86373 100644 --- a/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionBuilder.cs @@ -11,6 +11,10 @@ namespace ZeroLevel.Services.PartitionStorage public interface IStorePartitionBuilder : IStorePartitionBase { + long TotalRecords + { + get; + } IEnumerable> Iterate(); /// /// Writing a key-value pair diff --git a/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionMergeBuilder.cs b/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionMergeBuilder.cs index 2568a83..95c72a0 100644 --- a/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionMergeBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Interfaces/IStorePartitionMergeBuilder.cs @@ -9,6 +9,10 @@ public interface IStorePartitionMergeBuilder : IStorePartitionBase { + long TotalRecords + { + get; + } /// /// Writing a key-value pair /// diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs b/ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs index 5ccdb03..4fa445b 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs @@ -24,6 +24,11 @@ namespace ZeroLevel.Services.PartitionStorage private readonly Func _keyDeserializer; private readonly Func _valueDeserializer; + public long TotalRecords + { + get { return _temporaryAccessor.TotalRecords; } + } + /// /// Write catalog /// diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs index 5674c1f..95c89da 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using ZeroLevel.Services.PartitionStorage.Interfaces; @@ -16,6 +15,10 @@ namespace ZeroLevel.Services.PartitionStorage { private readonly Action _storeMethod; + private long _totalRecords = 0; + + public long TotalRecords { get { return _totalRecords; } } + public StorePartitionBuilder(StoreOptions options, TMeta info, IStoreSerializer serializer) @@ -38,11 +41,14 @@ namespace ZeroLevel.Services.PartitionStorage public void Store(TKey key, TInput value) { _storeMethod.Invoke(key, value); + Interlocked.Increment(ref _totalRecords); } + public void CompleteAdding() { CloseWriteStreams(); } + public void Compress() { var files = Directory.GetFiles(_catalog); diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index 1a91375..e14b011 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -6,7 +6,7 @@ ogoun ogoun - 3.3.8.3 + 3.3.8.4 PartitionStorage. ThreadSafeWriting option https://github.com/ogoun/Zero/wiki Copyright Ogoun 2022 @@ -14,8 +14,8 @@ https://github.com/ogoun/Zero git - 3.3.8.3 - 3.3.8.3 + 3.3.8.4 + 3.3.8.4 AnyCPU;x64;x86 zero.png full