ogoun
/
Zero
mirror of https://github.com/ogoun/Zero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix merge/remove operations

Browse Source
pull/4/head
Ogoun 2 years ago
parent 94d3e1ca71
commit d2ef4087af
3 changed files with 66 additions and 56 deletions
Show Stats Download Patch File Download Diff File

24
PartitionFileStorageTest/Program.cs
Unescape View File

@ -171,11 +171,11 @@ namespace PartitionFileStorageTest
var t = Generate(r);
storePart.Store(s, t);
}
if (s % 11217 == 0)
if (s % 177 == 0)
{
testKeys1.Add(s);
}
if (s % 11219 == 0)
if (s % 223 == 0)
{
testKeys2.Add(s);
}
@ -228,10 +228,7 @@ namespace PartitionFileStorageTest
Console.WriteLine("Test #1 remove by keys");
for (int i = 0; i < testKeys1.Count; i++)
{
if (i % 100 == 0)
{
readPart.RemoveKey(testKeys1[i]);
}
readPart.RemoveKey(testKeys1[i]);
}
Console.WriteLine("Test #1 reading after remove");
foreach (var key in testKeys1)
@ -359,8 +356,8 @@ namespace PartitionFileStorageTest
static void Main(string[] args)
{
var root = @"H:\temp";
SmallFullTest(root);
//TestBuildRemoveStore(root);
//SmallFullTest(root);
TestBuildRemoveStore(root);
//BuildStore(root);
//TestReading(root);
//TestIterations(root);
@ -371,20 +368,15 @@ namespace PartitionFileStorageTest
private static void TestRangeCompressionAndInversion()
{
var list = new List<FilePositionRange>();
list.Add(new FilePositionRange { Start = 1, End = 36 });
list.Add(new FilePositionRange { Start = 36, End = 63 });
list.Add(new FilePositionRange { Start = 63, End = 89 });
list.Add(new FilePositionRange { Start = 93, End = 118 });
list.Add(new FilePositionRange { Start = 126, End = 199 });
list.Add(new FilePositionRange { Start = 199, End = 216 });
list.Add(new FilePositionRange { Start = 277, End = 500 });
list.Add(new FilePositionRange { Start = 5, End = 12 });
list.Add(new FilePositionRange { Start = 16, End = 21 });
RangeCompression(list);
foreach (var r in list)
{
Console.WriteLine($"{r.Start}: {r.End}");
}
Console.WriteLine("Invert ranges");
var inverted = RangeInversion(list, 500);
var inverted = RangeInversion(list, 21);
foreach (var r in inverted)
{
Console.WriteLine($"{r.Start}: {r.End}");

88
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs
Unescape View File

@ -12,6 +12,7 @@ namespace ZeroLevel.Services.PartitionStorage
{
private readonly StoreOptions<TKey, TInput, TValue, TMeta> _options;
private readonly string _catalog;
private readonly string _indexCatalog;
private readonly TMeta _info;
public string Catalog { get { return _catalog; } }
@ -21,6 +22,10 @@ namespace ZeroLevel.Services.PartitionStorage
_info = info;
_options = options;
_catalog = _options.GetCatalogPath(info);
if (_options.Index.Enabled)
{
_indexCatalog = Path.Combine(_catalog, "__indexes__");
}
}
public int CountDataFiles() => Directory.GetFiles(_catalog)?.Length ?? 0;
@ -72,7 +77,7 @@ namespace ZeroLevel.Services.PartitionStorage
}
public IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> Find(IEnumerable<TKey> keys)
{
var results = keys
var results = keys.Distinct()
.GroupBy(
k => _options.GetFileName(k, _info),
k => k, (key, g) => new { FileName = key, Keys = g.ToArray() });
@ -123,41 +128,48 @@ namespace ZeroLevel.Services.PartitionStorage
{
if (_options.Index.Enabled)
{
var indexFolder = Path.Combine(_catalog, "__indexes__");
FSUtils.CleanAndTestFolder(indexFolder);
FSUtils.CleanAndTestFolder(_indexCatalog);
var files = Directory.GetFiles(_catalog);
if (files != null && files.Length > 0)
{
var dict = new Dictionary<TKey, long>();
foreach (var file in files)
{
dict.Clear();
using (var reader = GetReadStream(Path.GetFileName(file)))
{
while (reader.EOS == false)
{
var pos = reader.Stream.Position;
var k = reader.ReadCompatible<TKey>();
dict[k] = pos;
reader.ReadCompatible<TValue>();
}
}
if (dict.Count > _options.Index.FileIndexCount * 8)
{
var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero);
var index_file = Path.Combine(indexFolder, Path.GetFileName(file));
var d_arr = dict.OrderBy(p => p.Key).ToArray();
using (var writer = new MemoryStreamWriter(
new FileStream(index_file, FileMode.Create, FileAccess.Write, FileShare.None)))
{
for (int i = 0; i < _options.Index.FileIndexCount; i++)
{
var pair = d_arr[i * step];
writer.WriteCompatible(pair.Key);
writer.WriteLong(pair.Value);
}
}
}
RebuildFileIndex(file);
}
}
}
}
private void RebuildFileIndex(string file)
{
if (false == Directory.Exists(_indexCatalog))
{
Directory.CreateDirectory(_indexCatalog);
}
var dict = new Dictionary<TKey, long>();
using (var reader = GetReadStream(Path.GetFileName(file)))
{
while (reader.EOS == false)
{
var pos = reader.Stream.Position;
var k = reader.ReadCompatible<TKey>();
dict[k] = pos;
reader.ReadCompatible<TValue>();
}
}
if (dict.Count > _options.Index.FileIndexCount * 8)
{
var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero);
var index_file = Path.Combine(_indexCatalog, Path.GetFileName(file));
var d_arr = dict.OrderBy(p => p.Key).ToArray();
using (var writer = new MemoryStreamWriter(
new FileStream(index_file, FileMode.Create, FileAccess.Write, FileShare.None)))
{
for (int i = 0; i < _options.Index.FileIndexCount; i++)
{
var pair = d_arr[i * step];
writer.WriteCompatible(pair.Key);
writer.WriteLong(pair.Value);
}
}
}
@ -259,10 +271,10 @@ namespace ZeroLevel.Services.PartitionStorage
public void RemoveAllExceptKeys(IEnumerable<TKey> keys)
{
var results = keys
var results = keys.Distinct()
.GroupBy(
k => _options.GetFileName(k, _info),
k => k, (key, g) => new { FileName = key, Keys = g.ToArray() });
k => k, (key, g) => new { FileName = key, Keys = g.OrderBy(k => k).ToArray() });
foreach (var group in results)
{
RemoveKeyGroup(group.FileName, group.Keys, false);
@ -276,10 +288,10 @@ namespace ZeroLevel.Services.PartitionStorage
public void RemoveKeys(IEnumerable<TKey> keys)
{
var results = keys
var results = keys.Distinct()
.GroupBy(
k => _options.GetFileName(k, _info),
k => k, (key, g) => new { FileName = key, Keys = g.ToArray() });
k => k, (key, g) => new { FileName = key, Keys = g.OrderBy(k => k).ToArray() });
foreach (var group in results)
{
RemoveKeyGroup(group.FileName, group.Keys, true);
@ -388,6 +400,12 @@ namespace ZeroLevel.Services.PartitionStorage
// 3. Replace from temporary to original
File.Move(tempFile, filePath, true);
// Rebuild index if needs
if (_options.Index.Enabled)
{
RebuildFileIndex(filePath);
}
}
}

10
ZeroLevel/ZeroLevel.csproj
Unescape View File

@ -2,20 +2,20 @@
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<Description>Infrastructure layer tool set.
<Description>Infrastructure Layer Toolkit.
</Description>
<Authors>ogoun</Authors>
<Company>ogoun</Company>
<AssemblyVersion>3.3.7.6</AssemblyVersion>
<PackageReleaseNotes>PartitionStorage update indexes</PackageReleaseNotes>
<AssemblyVersion>3.3.7.7</AssemblyVersion>
<PackageReleaseNotes>PartitionStorage Merge, Remove and RemoveAllExcept methods</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2022</Copyright>
<PackageLicenseUrl></PackageLicenseUrl>
<PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<Version>3.3.7.6</Version>
<FileVersion>3.3.7.6</FileVersion>
<Version>3.3.7.7</Version>
<FileVersion>3.3.7.7</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType>

Write Preview
Loading…
Cancel
Save

Powered by TurnKey Linux.