ogoun
/
Zero
mirror of https://github.com/ogoun/Zero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix merge/remove operations

Browse Source
pull/4/head
Ogoun 2 years ago
parent 94d3e1ca71
commit d2ef4087af
3 changed files with 66 additions and 56 deletions
Show Stats Download Patch File Download Diff File

22
PartitionFileStorageTest/Program.cs
Unescape View File

@ -171,11 +171,11 @@ namespace PartitionFileStorageTest
var t = Generate(r); var t = Generate(r);
storePart.Store(s, t); storePart.Store(s, t);
} }
if (s % 11217 == 0) if (s % 177 == 0)
{ {
testKeys1.Add(s); testKeys1.Add(s);
} }
if (s % 11219 == 0) if (s % 223 == 0)
{ {
testKeys2.Add(s); testKeys2.Add(s);
} }
@ -227,12 +227,9 @@ namespace PartitionFileStorageTest
Console.ReadKey(); Console.ReadKey();
Console.WriteLine("Test #1 remove by keys"); Console.WriteLine("Test #1 remove by keys");
for (int i = 0; i < testKeys1.Count; i++) for (int i = 0; i < testKeys1.Count; i++)
{
if (i % 100 == 0)
{ {
readPart.RemoveKey(testKeys1[i]); readPart.RemoveKey(testKeys1[i]);
} }
}
Console.WriteLine("Test #1 reading after remove"); Console.WriteLine("Test #1 reading after remove");
foreach (var key in testKeys1) foreach (var key in testKeys1)
{ {
@ -359,8 +356,8 @@ namespace PartitionFileStorageTest
static void Main(string[] args) static void Main(string[] args)
{ {
var root = @"H:\temp"; var root = @"H:\temp";
SmallFullTest(root); //SmallFullTest(root);
//TestBuildRemoveStore(root); TestBuildRemoveStore(root);
//BuildStore(root); //BuildStore(root);
//TestReading(root); //TestReading(root);
//TestIterations(root); //TestIterations(root);
@ -371,20 +368,15 @@ namespace PartitionFileStorageTest
private static void TestRangeCompressionAndInversion() private static void TestRangeCompressionAndInversion()
{ {
var list = new List<FilePositionRange>(); var list = new List<FilePositionRange>();
list.Add(new FilePositionRange { Start = 1, End = 36 }); list.Add(new FilePositionRange { Start = 5, End = 12 });
list.Add(new FilePositionRange { Start = 36, End = 63 }); list.Add(new FilePositionRange { Start = 16, End = 21 });
list.Add(new FilePositionRange { Start = 63, End = 89 });
list.Add(new FilePositionRange { Start = 93, End = 118 });
list.Add(new FilePositionRange { Start = 126, End = 199 });
list.Add(new FilePositionRange { Start = 199, End = 216 });
list.Add(new FilePositionRange { Start = 277, End = 500 });
RangeCompression(list); RangeCompression(list);
foreach (var r in list) foreach (var r in list)
{ {
Console.WriteLine($"{r.Start}: {r.End}"); Console.WriteLine($"{r.Start}: {r.End}");
} }
Console.WriteLine("Invert ranges"); Console.WriteLine("Invert ranges");
var inverted = RangeInversion(list, 500); var inverted = RangeInversion(list, 21);
foreach (var r in inverted) foreach (var r in inverted)
{ {
Console.WriteLine($"{r.Start}: {r.End}"); Console.WriteLine($"{r.Start}: {r.End}");

44
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs
Unescape View File

@ -12,6 +12,7 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
private readonly StoreOptions<TKey, TInput, TValue, TMeta> _options; private readonly StoreOptions<TKey, TInput, TValue, TMeta> _options;
private readonly string _catalog; private readonly string _catalog;
private readonly string _indexCatalog;
private readonly TMeta _info; private readonly TMeta _info;
public string Catalog { get { return _catalog; } } public string Catalog { get { return _catalog; } }
@ -21,6 +22,10 @@ namespace ZeroLevel.Services.PartitionStorage
_info = info; _info = info;
_options = options; _options = options;
_catalog = _options.GetCatalogPath(info); _catalog = _options.GetCatalogPath(info);
if (_options.Index.Enabled)
{
_indexCatalog = Path.Combine(_catalog, "__indexes__");
}
} }
public int CountDataFiles() => Directory.GetFiles(_catalog)?.Length ?? 0; public int CountDataFiles() => Directory.GetFiles(_catalog)?.Length ?? 0;
@ -72,7 +77,7 @@ namespace ZeroLevel.Services.PartitionStorage
} }
public IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> Find(IEnumerable<TKey> keys) public IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> Find(IEnumerable<TKey> keys)
{ {
var results = keys var results = keys.Distinct()
.GroupBy( .GroupBy(
k => _options.GetFileName(k, _info), k => _options.GetFileName(k, _info),
k => k, (key, g) => new { FileName = key, Keys = g.ToArray() }); k => k, (key, g) => new { FileName = key, Keys = g.ToArray() });
@ -123,15 +128,25 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
if (_options.Index.Enabled) if (_options.Index.Enabled)
{ {
var indexFolder = Path.Combine(_catalog, "__indexes__"); FSUtils.CleanAndTestFolder(_indexCatalog);
FSUtils.CleanAndTestFolder(indexFolder);
var files = Directory.GetFiles(_catalog); var files = Directory.GetFiles(_catalog);
if (files != null && files.Length > 0) if (files != null && files.Length > 0)
{ {
var dict = new Dictionary<TKey, long>();
foreach (var file in files) foreach (var file in files)
{ {
dict.Clear(); RebuildFileIndex(file);
}
}
}
}
private void RebuildFileIndex(string file)
{
if (false == Directory.Exists(_indexCatalog))
{
Directory.CreateDirectory(_indexCatalog);
}
var dict = new Dictionary<TKey, long>();
using (var reader = GetReadStream(Path.GetFileName(file))) using (var reader = GetReadStream(Path.GetFileName(file)))
{ {
while (reader.EOS == false) while (reader.EOS == false)
@ -145,7 +160,7 @@ namespace ZeroLevel.Services.PartitionStorage
if (dict.Count > _options.Index.FileIndexCount * 8) if (dict.Count > _options.Index.FileIndexCount * 8)
{ {
var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero); var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero);
var index_file = Path.Combine(indexFolder, Path.GetFileName(file)); var index_file = Path.Combine(_indexCatalog, Path.GetFileName(file));
var d_arr = dict.OrderBy(p => p.Key).ToArray(); var d_arr = dict.OrderBy(p => p.Key).ToArray();
using (var writer = new MemoryStreamWriter( using (var writer = new MemoryStreamWriter(
new FileStream(index_file, FileMode.Create, FileAccess.Write, FileShare.None))) new FileStream(index_file, FileMode.Create, FileAccess.Write, FileShare.None)))
@ -159,9 +174,6 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
}
}
}
#region Private methods #region Private methods
private IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> Find(string fileName, private IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> Find(string fileName,
@ -259,10 +271,10 @@ namespace ZeroLevel.Services.PartitionStorage
public void RemoveAllExceptKeys(IEnumerable<TKey> keys) public void RemoveAllExceptKeys(IEnumerable<TKey> keys)
{ {
var results = keys var results = keys.Distinct()
.GroupBy( .GroupBy(
k => _options.GetFileName(k, _info), k => _options.GetFileName(k, _info),
k => k, (key, g) => new { FileName = key, Keys = g.ToArray() }); k => k, (key, g) => new { FileName = key, Keys = g.OrderBy(k => k).ToArray() });
foreach (var group in results) foreach (var group in results)
{ {
RemoveKeyGroup(group.FileName, group.Keys, false); RemoveKeyGroup(group.FileName, group.Keys, false);
@ -276,10 +288,10 @@ namespace ZeroLevel.Services.PartitionStorage
public void RemoveKeys(IEnumerable<TKey> keys) public void RemoveKeys(IEnumerable<TKey> keys)
{ {
var results = keys var results = keys.Distinct()
.GroupBy( .GroupBy(
k => _options.GetFileName(k, _info), k => _options.GetFileName(k, _info),
k => k, (key, g) => new { FileName = key, Keys = g.ToArray() }); k => k, (key, g) => new { FileName = key, Keys = g.OrderBy(k => k).ToArray() });
foreach (var group in results) foreach (var group in results)
{ {
RemoveKeyGroup(group.FileName, group.Keys, true); RemoveKeyGroup(group.FileName, group.Keys, true);
@ -388,6 +400,12 @@ namespace ZeroLevel.Services.PartitionStorage
// 3. Replace from temporary to original // 3. Replace from temporary to original
File.Move(tempFile, filePath, true); File.Move(tempFile, filePath, true);
// Rebuild index if needs
if (_options.Index.Enabled)
{
RebuildFileIndex(filePath);
}
} }
} }

10
ZeroLevel/ZeroLevel.csproj
Unescape View File

@ -2,20 +2,20 @@
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net6.0</TargetFramework>
<Description>Infrastructure layer tool set. <Description>Infrastructure Layer Toolkit.
</Description> </Description>
<Authors>ogoun</Authors> <Authors>ogoun</Authors>
<Company>ogoun</Company> <Company>ogoun</Company>
<AssemblyVersion>3.3.7.6</AssemblyVersion> <AssemblyVersion>3.3.7.7</AssemblyVersion>
<PackageReleaseNotes>PartitionStorage update indexes</PackageReleaseNotes> <PackageReleaseNotes>PartitionStorage Merge, Remove and RemoveAllExcept methods</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl> <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2022</Copyright> <Copyright>Copyright Ogoun 2022</Copyright>
<PackageLicenseUrl></PackageLicenseUrl> <PackageLicenseUrl></PackageLicenseUrl>
<PackageIconUrl></PackageIconUrl> <PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl> <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType> <RepositoryType>git</RepositoryType>
<Version>3.3.7.6</Version> <Version>3.3.7.7</Version>
<FileVersion>3.3.7.6</FileVersion> <FileVersion>3.3.7.7</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms> <Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon> <PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType> <DebugType>full</DebugType>

Write Preview
Loading…
Cancel
Save

Powered by TurnKey Linux.