ogoun
/
Zero
mirror of https://github.com/ogoun/Zero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix merge/remove operations

Browse Source
pull/4/head
Ogoun 2 years ago
parent 94d3e1ca71
commit d2ef4087af
3 changed files with 66 additions and 56 deletions
Show Stats Download Patch File Download Diff File

24
PartitionFileStorageTest/Program.cs
Unescape View File

@ -171,11 +171,11 @@ namespace PartitionFileStorageTest
var t = Generate(r); var t = Generate(r);
storePart.Store(s, t); storePart.Store(s, t);
} }
if (s % 11217 == 0) if (s % 177 == 0)
{ {
testKeys1.Add(s); testKeys1.Add(s);
} }
if (s % 11219 == 0) if (s % 223 == 0)
{ {
testKeys2.Add(s); testKeys2.Add(s);
} }
@ -228,10 +228,7 @@ namespace PartitionFileStorageTest
Console.WriteLine("Test #1 remove by keys"); Console.WriteLine("Test #1 remove by keys");
for (int i = 0; i < testKeys1.Count; i++) for (int i = 0; i < testKeys1.Count; i++)
{ {
if (i % 100 == 0) readPart.RemoveKey(testKeys1[i]);
{
readPart.RemoveKey(testKeys1[i]);
}
} }
Console.WriteLine("Test #1 reading after remove"); Console.WriteLine("Test #1 reading after remove");
foreach (var key in testKeys1) foreach (var key in testKeys1)
@ -359,8 +356,8 @@ namespace PartitionFileStorageTest
static void Main(string[] args) static void Main(string[] args)
{ {
var root = @"H:\temp"; var root = @"H:\temp";
SmallFullTest(root); //SmallFullTest(root);
//TestBuildRemoveStore(root); TestBuildRemoveStore(root);
//BuildStore(root); //BuildStore(root);
//TestReading(root); //TestReading(root);
//TestIterations(root); //TestIterations(root);
@ -371,20 +368,15 @@ namespace PartitionFileStorageTest
private static void TestRangeCompressionAndInversion() private static void TestRangeCompressionAndInversion()
{ {
var list = new List<FilePositionRange>(); var list = new List<FilePositionRange>();
list.Add(new FilePositionRange { Start = 1, End = 36 }); list.Add(new FilePositionRange { Start = 5, End = 12 });
list.Add(new FilePositionRange { Start = 36, End = 63 }); list.Add(new FilePositionRange { Start = 16, End = 21 });
list.Add(new FilePositionRange { Start = 63, End = 89 });
list.Add(new FilePositionRange { Start = 93, End = 118 });
list.Add(new FilePositionRange { Start = 126, End = 199 });
list.Add(new FilePositionRange { Start = 199, End = 216 });
list.Add(new FilePositionRange { Start = 277, End = 500 });
RangeCompression(list); RangeCompression(list);
foreach (var r in list) foreach (var r in list)
{ {
Console.WriteLine($"{r.Start}: {r.End}"); Console.WriteLine($"{r.Start}: {r.End}");
} }
Console.WriteLine("Invert ranges"); Console.WriteLine("Invert ranges");
var inverted = RangeInversion(list, 500); var inverted = RangeInversion(list, 21);
foreach (var r in inverted) foreach (var r in inverted)
{ {
Console.WriteLine($"{r.Start}: {r.End}"); Console.WriteLine($"{r.Start}: {r.End}");

88
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs
Unescape View File

@ -12,6 +12,7 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
private readonly StoreOptions<TKey, TInput, TValue, TMeta> _options; private readonly StoreOptions<TKey, TInput, TValue, TMeta> _options;
private readonly string _catalog; private readonly string _catalog;
private readonly string _indexCatalog;
private readonly TMeta _info; private readonly TMeta _info;
public string Catalog { get { return _catalog; } } public string Catalog { get { return _catalog; } }
@ -21,6 +22,10 @@ namespace ZeroLevel.Services.PartitionStorage
_info = info; _info = info;
_options = options; _options = options;
_catalog = _options.GetCatalogPath(info); _catalog = _options.GetCatalogPath(info);
if (_options.Index.Enabled)
{
_indexCatalog = Path.Combine(_catalog, "__indexes__");
}
} }
public int CountDataFiles() => Directory.GetFiles(_catalog)?.Length ?? 0; public int CountDataFiles() => Directory.GetFiles(_catalog)?.Length ?? 0;
@ -72,7 +77,7 @@ namespace ZeroLevel.Services.PartitionStorage
} }
public IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> Find(IEnumerable<TKey> keys) public IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> Find(IEnumerable<TKey> keys)
{ {
var results = keys var results = keys.Distinct()
.GroupBy( .GroupBy(
k => _options.GetFileName(k, _info), k => _options.GetFileName(k, _info),
k => k, (key, g) => new { FileName = key, Keys = g.ToArray() }); k => k, (key, g) => new { FileName = key, Keys = g.ToArray() });
@ -123,41 +128,48 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
if (_options.Index.Enabled) if (_options.Index.Enabled)
{ {
var indexFolder = Path.Combine(_catalog, "__indexes__"); FSUtils.CleanAndTestFolder(_indexCatalog);
FSUtils.CleanAndTestFolder(indexFolder);
var files = Directory.GetFiles(_catalog); var files = Directory.GetFiles(_catalog);
if (files != null && files.Length > 0) if (files != null && files.Length > 0)
{ {
var dict = new Dictionary<TKey, long>();
foreach (var file in files) foreach (var file in files)
{ {
dict.Clear(); RebuildFileIndex(file);
using (var reader = GetReadStream(Path.GetFileName(file))) }
{ }
while (reader.EOS == false) }
{ }
var pos = reader.Stream.Position;
var k = reader.ReadCompatible<TKey>(); private void RebuildFileIndex(string file)
dict[k] = pos; {
reader.ReadCompatible<TValue>(); if (false == Directory.Exists(_indexCatalog))
} {
} Directory.CreateDirectory(_indexCatalog);
if (dict.Count > _options.Index.FileIndexCount * 8) }
{ var dict = new Dictionary<TKey, long>();
var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero); using (var reader = GetReadStream(Path.GetFileName(file)))
var index_file = Path.Combine(indexFolder, Path.GetFileName(file)); {
var d_arr = dict.OrderBy(p => p.Key).ToArray(); while (reader.EOS == false)
using (var writer = new MemoryStreamWriter( {
new FileStream(index_file, FileMode.Create, FileAccess.Write, FileShare.None))) var pos = reader.Stream.Position;
{ var k = reader.ReadCompatible<TKey>();
for (int i = 0; i < _options.Index.FileIndexCount; i++) dict[k] = pos;
{ reader.ReadCompatible<TValue>();
var pair = d_arr[i * step]; }
writer.WriteCompatible(pair.Key); }
writer.WriteLong(pair.Value); if (dict.Count > _options.Index.FileIndexCount * 8)
} {
} var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero);
} var index_file = Path.Combine(_indexCatalog, Path.GetFileName(file));
var d_arr = dict.OrderBy(p => p.Key).ToArray();
using (var writer = new MemoryStreamWriter(
new FileStream(index_file, FileMode.Create, FileAccess.Write, FileShare.None)))
{
for (int i = 0; i < _options.Index.FileIndexCount; i++)
{
var pair = d_arr[i * step];
writer.WriteCompatible(pair.Key);
writer.WriteLong(pair.Value);
} }
} }
} }
@ -259,10 +271,10 @@ namespace ZeroLevel.Services.PartitionStorage
public void RemoveAllExceptKeys(IEnumerable<TKey> keys) public void RemoveAllExceptKeys(IEnumerable<TKey> keys)
{ {
var results = keys var results = keys.Distinct()
.GroupBy( .GroupBy(
k => _options.GetFileName(k, _info), k => _options.GetFileName(k, _info),
k => k, (key, g) => new { FileName = key, Keys = g.ToArray() }); k => k, (key, g) => new { FileName = key, Keys = g.OrderBy(k => k).ToArray() });
foreach (var group in results) foreach (var group in results)
{ {
RemoveKeyGroup(group.FileName, group.Keys, false); RemoveKeyGroup(group.FileName, group.Keys, false);
@ -276,10 +288,10 @@ namespace ZeroLevel.Services.PartitionStorage
public void RemoveKeys(IEnumerable<TKey> keys) public void RemoveKeys(IEnumerable<TKey> keys)
{ {
var results = keys var results = keys.Distinct()
.GroupBy( .GroupBy(
k => _options.GetFileName(k, _info), k => _options.GetFileName(k, _info),
k => k, (key, g) => new { FileName = key, Keys = g.ToArray() }); k => k, (key, g) => new { FileName = key, Keys = g.OrderBy(k => k).ToArray() });
foreach (var group in results) foreach (var group in results)
{ {
RemoveKeyGroup(group.FileName, group.Keys, true); RemoveKeyGroup(group.FileName, group.Keys, true);
@ -388,6 +400,12 @@ namespace ZeroLevel.Services.PartitionStorage
// 3. Replace from temporary to original // 3. Replace from temporary to original
File.Move(tempFile, filePath, true); File.Move(tempFile, filePath, true);
// Rebuild index if needs
if (_options.Index.Enabled)
{
RebuildFileIndex(filePath);
}
} }
} }

10
ZeroLevel/ZeroLevel.csproj
Unescape View File

@ -2,20 +2,20 @@
<PropertyGroup> <PropertyGroup>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net6.0</TargetFramework>
<Description>Infrastructure layer tool set. <Description>Infrastructure Layer Toolkit.
</Description> </Description>
<Authors>ogoun</Authors> <Authors>ogoun</Authors>
<Company>ogoun</Company> <Company>ogoun</Company>
<AssemblyVersion>3.3.7.6</AssemblyVersion> <AssemblyVersion>3.3.7.7</AssemblyVersion>
<PackageReleaseNotes>PartitionStorage update indexes</PackageReleaseNotes> <PackageReleaseNotes>PartitionStorage Merge, Remove and RemoveAllExcept methods</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl> <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2022</Copyright> <Copyright>Copyright Ogoun 2022</Copyright>
<PackageLicenseUrl></PackageLicenseUrl> <PackageLicenseUrl></PackageLicenseUrl>
<PackageIconUrl></PackageIconUrl> <PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl> <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType> <RepositoryType>git</RepositoryType>
<Version>3.3.7.6</Version> <Version>3.3.7.7</Version>
<FileVersion>3.3.7.6</FileVersion> <FileVersion>3.3.7.7</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms> <Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon> <PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType> <DebugType>full</DebugType>

Write Preview
Loading…
Cancel
Save

Powered by TurnKey Linux.