From d2ef4087af35334a20ac8390154633ae3d8b8d4d Mon Sep 17 00:00:00 2001 From: Ogoun Date: Fri, 18 Nov 2022 01:35:42 +0300 Subject: [PATCH] Fix merge/remove operations --- PartitionFileStorageTest/Program.cs | 24 ++--- .../Partition/StorePartitionAccessor.cs | 88 +++++++++++-------- ZeroLevel/ZeroLevel.csproj | 10 +-- 3 files changed, 66 insertions(+), 56 deletions(-) diff --git a/PartitionFileStorageTest/Program.cs b/PartitionFileStorageTest/Program.cs index 8f5fe00..dcf7337 100644 --- a/PartitionFileStorageTest/Program.cs +++ b/PartitionFileStorageTest/Program.cs @@ -171,11 +171,11 @@ namespace PartitionFileStorageTest var t = Generate(r); storePart.Store(s, t); } - if (s % 11217 == 0) + if (s % 177 == 0) { testKeys1.Add(s); } - if (s % 11219 == 0) + if (s % 223 == 0) { testKeys2.Add(s); } @@ -228,10 +228,7 @@ namespace PartitionFileStorageTest Console.WriteLine("Test #1 remove by keys"); for (int i = 0; i < testKeys1.Count; i++) { - if (i % 100 == 0) - { - readPart.RemoveKey(testKeys1[i]); - } + readPart.RemoveKey(testKeys1[i]); } Console.WriteLine("Test #1 reading after remove"); foreach (var key in testKeys1) @@ -359,8 +356,8 @@ namespace PartitionFileStorageTest static void Main(string[] args) { var root = @"H:\temp"; - SmallFullTest(root); - //TestBuildRemoveStore(root); + //SmallFullTest(root); + TestBuildRemoveStore(root); //BuildStore(root); //TestReading(root); //TestIterations(root); @@ -371,20 +368,15 @@ namespace PartitionFileStorageTest private static void TestRangeCompressionAndInversion() { var list = new List(); - list.Add(new FilePositionRange { Start = 1, End = 36 }); - list.Add(new FilePositionRange { Start = 36, End = 63 }); - list.Add(new FilePositionRange { Start = 63, End = 89 }); - list.Add(new FilePositionRange { Start = 93, End = 118 }); - list.Add(new FilePositionRange { Start = 126, End = 199 }); - list.Add(new FilePositionRange { Start = 199, End = 216 }); - list.Add(new FilePositionRange { Start = 277, End = 500 }); + list.Add(new FilePositionRange { Start = 5, End = 12 }); + list.Add(new FilePositionRange { Start = 16, End = 21 }); RangeCompression(list); foreach (var r in list) { Console.WriteLine($"{r.Start}: {r.End}"); } Console.WriteLine("Invert ranges"); - var inverted = RangeInversion(list, 500); + var inverted = RangeInversion(list, 21); foreach (var r in inverted) { Console.WriteLine($"{r.Start}: {r.End}"); diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs index e4f3c7d..6661511 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs @@ -12,6 +12,7 @@ namespace ZeroLevel.Services.PartitionStorage { private readonly StoreOptions _options; private readonly string _catalog; + private readonly string _indexCatalog; private readonly TMeta _info; public string Catalog { get { return _catalog; } } @@ -21,6 +22,10 @@ namespace ZeroLevel.Services.PartitionStorage _info = info; _options = options; _catalog = _options.GetCatalogPath(info); + if (_options.Index.Enabled) + { + _indexCatalog = Path.Combine(_catalog, "__indexes__"); + } } public int CountDataFiles() => Directory.GetFiles(_catalog)?.Length ?? 0; @@ -72,7 +77,7 @@ namespace ZeroLevel.Services.PartitionStorage } public IEnumerable> Find(IEnumerable keys) { - var results = keys + var results = keys.Distinct() .GroupBy( k => _options.GetFileName(k, _info), k => k, (key, g) => new { FileName = key, Keys = g.ToArray() }); @@ -123,41 +128,48 @@ namespace ZeroLevel.Services.PartitionStorage { if (_options.Index.Enabled) { - var indexFolder = Path.Combine(_catalog, "__indexes__"); - FSUtils.CleanAndTestFolder(indexFolder); + FSUtils.CleanAndTestFolder(_indexCatalog); var files = Directory.GetFiles(_catalog); if (files != null && files.Length > 0) { - var dict = new Dictionary(); foreach (var file in files) { - dict.Clear(); - using (var reader = GetReadStream(Path.GetFileName(file))) - { - while (reader.EOS == false) - { - var pos = reader.Stream.Position; - var k = reader.ReadCompatible(); - dict[k] = pos; - reader.ReadCompatible(); - } - } - if (dict.Count > _options.Index.FileIndexCount * 8) - { - var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero); - var index_file = Path.Combine(indexFolder, Path.GetFileName(file)); - var d_arr = dict.OrderBy(p => p.Key).ToArray(); - using (var writer = new MemoryStreamWriter( - new FileStream(index_file, FileMode.Create, FileAccess.Write, FileShare.None))) - { - for (int i = 0; i < _options.Index.FileIndexCount; i++) - { - var pair = d_arr[i * step]; - writer.WriteCompatible(pair.Key); - writer.WriteLong(pair.Value); - } - } - } + RebuildFileIndex(file); + } + } + } + } + + private void RebuildFileIndex(string file) + { + if (false == Directory.Exists(_indexCatalog)) + { + Directory.CreateDirectory(_indexCatalog); + } + var dict = new Dictionary(); + using (var reader = GetReadStream(Path.GetFileName(file))) + { + while (reader.EOS == false) + { + var pos = reader.Stream.Position; + var k = reader.ReadCompatible(); + dict[k] = pos; + reader.ReadCompatible(); + } + } + if (dict.Count > _options.Index.FileIndexCount * 8) + { + var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero); + var index_file = Path.Combine(_indexCatalog, Path.GetFileName(file)); + var d_arr = dict.OrderBy(p => p.Key).ToArray(); + using (var writer = new MemoryStreamWriter( + new FileStream(index_file, FileMode.Create, FileAccess.Write, FileShare.None))) + { + for (int i = 0; i < _options.Index.FileIndexCount; i++) + { + var pair = d_arr[i * step]; + writer.WriteCompatible(pair.Key); + writer.WriteLong(pair.Value); } } } @@ -259,10 +271,10 @@ namespace ZeroLevel.Services.PartitionStorage public void RemoveAllExceptKeys(IEnumerable keys) { - var results = keys + var results = keys.Distinct() .GroupBy( k => _options.GetFileName(k, _info), - k => k, (key, g) => new { FileName = key, Keys = g.ToArray() }); + k => k, (key, g) => new { FileName = key, Keys = g.OrderBy(k => k).ToArray() }); foreach (var group in results) { RemoveKeyGroup(group.FileName, group.Keys, false); @@ -276,10 +288,10 @@ namespace ZeroLevel.Services.PartitionStorage public void RemoveKeys(IEnumerable keys) { - var results = keys + var results = keys.Distinct() .GroupBy( k => _options.GetFileName(k, _info), - k => k, (key, g) => new { FileName = key, Keys = g.ToArray() }); + k => k, (key, g) => new { FileName = key, Keys = g.OrderBy(k => k).ToArray() }); foreach (var group in results) { RemoveKeyGroup(group.FileName, group.Keys, true); @@ -388,6 +400,12 @@ namespace ZeroLevel.Services.PartitionStorage // 3. Replace from temporary to original File.Move(tempFile, filePath, true); + + // Rebuild index if needs + if (_options.Index.Enabled) + { + RebuildFileIndex(filePath); + } } } diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index b3653c0..5d09671 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -2,20 +2,20 @@ net6.0 - Infrastructure layer tool set. + Infrastructure Layer Toolkit. ogoun ogoun - 3.3.7.6 - PartitionStorage update indexes + 3.3.7.7 + PartitionStorage Merge, Remove and RemoveAllExcept methods https://github.com/ogoun/Zero/wiki Copyright Ogoun 2022 https://github.com/ogoun/Zero git - 3.3.7.6 - 3.3.7.6 + 3.3.7.7 + 3.3.7.7 AnyCPU;x64;x86 zero.png full