ogoun
/
Zero
mirror of https://github.com/ogoun/Zero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Safe open streams

Browse Source
pull/4/head
Ogoun 2 years ago
parent b3b40ade33
commit 7157eaa9ff
5 changed files with 242 additions and 170 deletions
Show Stats Download Patch File Download Diff File

5
ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs
Unescape View File

@ -126,12 +126,11 @@ namespace ZeroLevel.Services.PartitionStorage
var k = _keyDeserializer.Invoke(reader); var k = _keyDeserializer.Invoke(reader);
var v = _valueDeserializer.Invoke(reader); var v = _valueDeserializer.Invoke(reader);
var input = _decompress(v); var input = _decompress(v);
yield return yield return new StorePartitionKeyValueSearchResult<TKey, IEnumerable<TInput>>
new StorePartitionKeyValueSearchResult<TKey, IEnumerable<TInput>>
{ {
Key = k, Key = k,
Value = input, Value = input,
Found = true Status = SearchResult.Success
}; };
} }
} }

75
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs
Unescape View File

@ -49,7 +49,9 @@ namespace ZeroLevel.Services.PartitionStorage
var offset = index.GetOffset(key); var offset = index.GetOffset(key);
startOffset = offset.Offset; startOffset = offset.Offset;
} }
using (var reader = GetReadStream(fileName)) if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{ {
if (startOffset > 0) if (startOffset > 0)
{ {
@ -64,7 +66,7 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
Key = key, Key = key,
Value = v, Value = v,
Found = true Status = SearchResult.Success
}; };
if (c == -1) if (c == -1)
{ {
@ -73,10 +75,20 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
else
{
return new StorePartitionKeyValueSearchResult<TKey, TValue>
{
Key = key,
Status = SearchResult.FileLocked,
Value = default
};
}
}
return new StorePartitionKeyValueSearchResult<TKey, TValue> return new StorePartitionKeyValueSearchResult<TKey, TValue>
{ {
Key = key, Key = key,
Found = false, Status = SearchResult.NotFound,
Value = default Value = default
}; };
} }
@ -101,13 +113,16 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
foreach (var file in files) foreach (var file in files)
{ {
using (var reader = GetReadStream(Path.GetFileName(file))) if (TryGetReadStream(file, out var reader))
{
using (reader)
{ {
while (reader.EOS == false) while (reader.EOS == false)
{ {
var k = _keyDeserializer.Invoke(reader); var k = _keyDeserializer.Invoke(reader);
var v = _valueDeserializer.Invoke(reader); var v = _valueDeserializer.Invoke(reader);
yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Found = true }; yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Status = SearchResult.Success };
}
} }
} }
} }
@ -118,13 +133,16 @@ namespace ZeroLevel.Services.PartitionStorage
var fileName = _options.GetFileName(key, _info); var fileName = _options.GetFileName(key, _info);
if (File.Exists(Path.Combine(_catalog, fileName))) if (File.Exists(Path.Combine(_catalog, fileName)))
{ {
using (var reader = GetReadStream(fileName)) if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{ {
while (reader.EOS == false) while (reader.EOS == false)
{ {
var k = _keyDeserializer.Invoke(reader); var k = _keyDeserializer.Invoke(reader);
var v = _valueDeserializer.Invoke(reader); var v = _valueDeserializer.Invoke(reader);
yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Found = true }; yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Status = SearchResult.Success };
}
} }
} }
} }
@ -197,7 +215,9 @@ namespace ZeroLevel.Services.PartitionStorage
Directory.CreateDirectory(_indexCatalog); Directory.CreateDirectory(_indexCatalog);
} }
var dict = new Dictionary<TKey, long>(); var dict = new Dictionary<TKey, long>();
using (var reader = GetReadStream(Path.GetFileName(file))) if (TryGetReadStream(file, out var reader))
{
using (reader)
{ {
while (reader.EOS == false) while (reader.EOS == false)
{ {
@ -207,6 +227,7 @@ namespace ZeroLevel.Services.PartitionStorage
_valueDeserializer.Invoke(reader); _valueDeserializer.Invoke(reader);
} }
} }
}
if (dict.Count > _options.Index.FileIndexCount * 8) if (dict.Count > _options.Index.FileIndexCount * 8)
{ {
var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero); var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero);
@ -237,7 +258,9 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
var index = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, _options.FilePartition, _options.KeyComparer); var index = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, _options.FilePartition, _options.KeyComparer);
var offsets = index.GetOffset(keys, true); var offsets = index.GetOffset(keys, true);
using (var reader = GetReadStream(fileName)) if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{ {
for (int i = 0; i < keys.Length; i++) for (int i = 0; i < keys.Length; i++)
{ {
@ -255,7 +278,7 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
Key = searchKey, Key = searchKey,
Value = v, Value = v,
Found = true Status = SearchResult.Success
}; };
break; break;
} }
@ -267,9 +290,12 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
}
else else
{ {
using (var reader = GetReadStream(fileName)) if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{ {
int index = 0; int index = 0;
var keys_arr = keys.OrderBy(k => k).ToArray(); var keys_arr = keys.OrderBy(k => k).ToArray();
@ -284,7 +310,7 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
Key = keys_arr[index], Key = keys_arr[index],
Value = v, Value = v,
Found = true Status = SearchResult.Success
}; };
index++; index++;
} }
@ -304,6 +330,7 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
}
private void RemoveKeyGroup(string fileName, TKey[] keys, bool inverseRemove, bool autoReindex) private void RemoveKeyGroup(string fileName, TKey[] keys, bool inverseRemove, bool autoReindex)
{ {
@ -316,7 +343,9 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
var index = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, _options.FilePartition, _options.KeyComparer); var index = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, _options.FilePartition, _options.KeyComparer);
var offsets = index.GetOffset(keys, true); var offsets = index.GetOffset(keys, true);
using (var reader = GetReadStream(fileName)) if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{ {
for (int i = 0; i < keys.Length; i++) for (int i = 0; i < keys.Length; i++)
{ {
@ -342,9 +371,12 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
}
else else
{ {
using (var reader = GetReadStream(fileName)) if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{ {
int index = 0; int index = 0;
var keys_arr = keys.OrderBy(k => k).ToArray(); var keys_arr = keys.OrderBy(k => k).ToArray();
@ -374,6 +406,7 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
}
// 2. Temporary file from ranges // 2. Temporary file from ranges
var tempPath = Path.GetTempPath(); var tempPath = Path.GetTempPath();
@ -414,11 +447,21 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
private MemoryStreamReader GetReadStream(string fileName) private bool TryGetReadStream(string fileName, out MemoryStreamReader reader)
{
try
{ {
var filePath = Path.Combine(_catalog, fileName); var filePath = Path.Combine(_catalog, fileName);
var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096 * 1024); var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096 * 1024);
return new MemoryStreamReader(stream); reader = new MemoryStreamReader(stream);
return true;
}
catch (Exception ex)
{
Log.SystemError(ex, "[StorePartitionAccessor.TryGetReadStream]");
}
reader = null;
return false;
} }
#endregion #endregion

51
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs
Unescape View File

@ -52,11 +52,13 @@ namespace ZeroLevel.Services.PartitionStorage
public void Store(TKey key, TInput value) public void Store(TKey key, TInput value)
{ {
var fileName = _options.GetFileName(key, _info); var fileName = _options.GetFileName(key, _info);
var stream = GetWriteStream(fileName); if (TryGetWriteStream(fileName, out var stream))
{
_keySerializer.Invoke(stream, key); _keySerializer.Invoke(stream, key);
Thread.MemoryBarrier(); Thread.MemoryBarrier();
_inputSerializer.Invoke(stream, value); _inputSerializer.Invoke(stream, value);
} }
}
public void CompleteAdding() public void CompleteAdding()
{ {
// Close all write streams // Close all write streams
@ -86,18 +88,16 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
foreach (var file in files) foreach (var file in files)
{ {
using (var reader = GetReadStream(Path.GetFileName(file))) if (TryGetReadStream(file, out var reader))
{
using (reader)
{ {
while (reader.EOS == false) while (reader.EOS == false)
{ {
var key = _keyDeserializer.Invoke(reader); var key = _keyDeserializer.Invoke(reader);
if (reader.EOS)
{
yield return new StorePartitionKeyValueSearchResult<TKey, TInput> { Key = key, Value = default, Found = true };
break;
}
var val = _inputDeserializer.Invoke(reader); var val = _inputDeserializer.Invoke(reader);
yield return new StorePartitionKeyValueSearchResult<TKey, TInput> { Key = key, Value = val, Found = true }; yield return new StorePartitionKeyValueSearchResult<TKey, TInput> { Key = key, Value = val, Status = SearchResult.Success };
}
} }
} }
} }
@ -115,8 +115,9 @@ namespace ZeroLevel.Services.PartitionStorage
var dict = new Dictionary<TKey, long>(); var dict = new Dictionary<TKey, long>();
foreach (var file in files) foreach (var file in files)
{ {
dict.Clear(); if (TryGetReadStream(file, out var reader))
using (var reader = GetReadStream(Path.GetFileName(file))) {
using (reader)
{ {
while (reader.EOS == false) while (reader.EOS == false)
{ {
@ -144,6 +145,8 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
dict.Clear();
}
} }
} }
} }
@ -185,20 +188,40 @@ namespace ZeroLevel.Services.PartitionStorage
File.Delete(file); File.Delete(file);
File.Move(tempFile, file, true); File.Move(tempFile, file, true);
} }
private MemoryStreamWriter GetWriteStream(string fileName) private bool TryGetWriteStream(string fileName, out MemoryStreamWriter writer)
{
try
{ {
return _writeStreams.GetOrAdd(fileName, k => writer = _writeStreams.GetOrAdd(fileName, k =>
{ {
var filePath = Path.Combine(_catalog, k); var filePath = Path.Combine(_catalog, k);
var stream = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.None, 4096 * 1024); var stream = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.None, 4096 * 1024);
return new MemoryStreamWriter(stream); return new MemoryStreamWriter(stream);
}); });
return true;
}
catch (Exception ex)
{
Log.SystemError(ex, "[StorePartitionBuilder.TryGetWriteStream]");
}
writer = null;
return false;
} }
private MemoryStreamReader GetReadStream(string fileName) private bool TryGetReadStream(string fileName, out MemoryStreamReader reader)
{
try
{ {
var filePath = Path.Combine(_catalog, fileName); var filePath = Path.Combine(_catalog, fileName);
var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096 * 1024); var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096 * 1024);
return new MemoryStreamReader(stream); reader = new MemoryStreamReader(stream);
return true;
}
catch (Exception ex)
{
Log.SystemError(ex, "[StorePartitionBuilder.TryGetReadStream]");
}
reader = null;
return false;
} }
#endregion #endregion

9
ZeroLevel/Services/PartitionStorage/Search/StorePartitionKeyValueSearchResult.cs
Unescape View File

@ -1,8 +1,15 @@
namespace ZeroLevel.Services.PartitionStorage namespace ZeroLevel.Services.PartitionStorage
{ {
public enum SearchResult
{
Success,
NotFound,
FileLocked
}
public class StorePartitionKeyValueSearchResult<TKey, TValue> public class StorePartitionKeyValueSearchResult<TKey, TValue>
{ {
public bool Found { get; set; } public SearchResult Status { get; set; }
public TKey Key { get; set; } public TKey Key { get; set; }
public TValue Value { get; set; } public TValue Value { get; set; }
} }

6
ZeroLevel/ZeroLevel.csproj
Unescape View File

@ -6,7 +6,7 @@
</Description> </Description>
<Authors>ogoun</Authors> <Authors>ogoun</Authors>
<Company>ogoun</Company> <Company>ogoun</Company>
<AssemblyVersion>3.3.7.9</AssemblyVersion> <AssemblyVersion>3.3.8.1</AssemblyVersion>
<PackageReleaseNotes>PartitionStorage optimizations</PackageReleaseNotes> <PackageReleaseNotes>PartitionStorage optimizations</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl> <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2022</Copyright> <Copyright>Copyright Ogoun 2022</Copyright>
@ -14,8 +14,8 @@
<PackageIconUrl></PackageIconUrl> <PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl> <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType> <RepositoryType>git</RepositoryType>
<Version>3.3.7.9</Version> <Version>3.3.8.1</Version>
<FileVersion>3.3.7.9</FileVersion> <FileVersion>3.3.8.1</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms> <Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon> <PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType> <DebugType>full</DebugType>

Write Preview
Loading…
Cancel
Save

Powered by TurnKey Linux.