ogoun
/
Zero
mirror of https://github.com/ogoun/Zero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Safe open streams

Browse Source
pull/4/head
Ogoun 2 years ago
parent b3b40ade33
commit 7157eaa9ff
5 changed files with 242 additions and 170 deletions
Show Stats Download Patch File Download Diff File

5
ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs
Unescape View File

@ -126,12 +126,11 @@ namespace ZeroLevel.Services.PartitionStorage
var k = _keyDeserializer.Invoke(reader);
var v = _valueDeserializer.Invoke(reader);
var input = _decompress(v);
yield return
new StorePartitionKeyValueSearchResult<TKey, IEnumerable<TInput>>
yield return new StorePartitionKeyValueSearchResult<TKey, IEnumerable<TInput>>
{
Key = k,
Value = input,
Found = true
Status = SearchResult.Success
};
}
}

75
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs
Unescape View File

@ -49,7 +49,9 @@ namespace ZeroLevel.Services.PartitionStorage
var offset = index.GetOffset(key);
startOffset = offset.Offset;
}
using (var reader = GetReadStream(fileName))
if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{
if (startOffset > 0)
{
@ -64,7 +66,7 @@ namespace ZeroLevel.Services.PartitionStorage
{
Key = key,
Value = v,
Found = true
Status = SearchResult.Success
};
if (c == -1)
{
@ -73,10 +75,20 @@ namespace ZeroLevel.Services.PartitionStorage
}
}
}
else
{
return new StorePartitionKeyValueSearchResult<TKey, TValue>
{
Key = key,
Status = SearchResult.FileLocked,
Value = default
};
}
}
return new StorePartitionKeyValueSearchResult<TKey, TValue>
{
Key = key,
Found = false,
Status = SearchResult.NotFound,
Value = default
};
}
@ -101,13 +113,16 @@ namespace ZeroLevel.Services.PartitionStorage
{
foreach (var file in files)
{
using (var reader = GetReadStream(Path.GetFileName(file)))
if (TryGetReadStream(file, out var reader))
{
using (reader)
{
while (reader.EOS == false)
{
var k = _keyDeserializer.Invoke(reader);
var v = _valueDeserializer.Invoke(reader);
yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Found = true };
yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Status = SearchResult.Success };
}
}
}
}
@ -118,13 +133,16 @@ namespace ZeroLevel.Services.PartitionStorage
var fileName = _options.GetFileName(key, _info);
if (File.Exists(Path.Combine(_catalog, fileName)))
{
using (var reader = GetReadStream(fileName))
if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{
while (reader.EOS == false)
{
var k = _keyDeserializer.Invoke(reader);
var v = _valueDeserializer.Invoke(reader);
yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Found = true };
yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Status = SearchResult.Success };
}
}
}
}
@ -197,7 +215,9 @@ namespace ZeroLevel.Services.PartitionStorage
Directory.CreateDirectory(_indexCatalog);
}
var dict = new Dictionary<TKey, long>();
using (var reader = GetReadStream(Path.GetFileName(file)))
if (TryGetReadStream(file, out var reader))
{
using (reader)
{
while (reader.EOS == false)
{
@ -207,6 +227,7 @@ namespace ZeroLevel.Services.PartitionStorage
_valueDeserializer.Invoke(reader);
}
}
}
if (dict.Count > _options.Index.FileIndexCount * 8)
{
var step = (int)Math.Round(((float)dict.Count / (float)_options.Index.FileIndexCount), MidpointRounding.ToZero);
@ -237,7 +258,9 @@ namespace ZeroLevel.Services.PartitionStorage
{
var index = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, _options.FilePartition, _options.KeyComparer);
var offsets = index.GetOffset(keys, true);
using (var reader = GetReadStream(fileName))
if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{
for (int i = 0; i < keys.Length; i++)
{
@ -255,7 +278,7 @@ namespace ZeroLevel.Services.PartitionStorage
{
Key = searchKey,
Value = v,
Found = true
Status = SearchResult.Success
};
break;
}
@ -267,9 +290,12 @@ namespace ZeroLevel.Services.PartitionStorage
}
}
}
}
else
{
using (var reader = GetReadStream(fileName))
if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{
int index = 0;
var keys_arr = keys.OrderBy(k => k).ToArray();
@ -284,7 +310,7 @@ namespace ZeroLevel.Services.PartitionStorage
{
Key = keys_arr[index],
Value = v,
Found = true
Status = SearchResult.Success
};
index++;
}
@ -304,6 +330,7 @@ namespace ZeroLevel.Services.PartitionStorage
}
}
}
}
private void RemoveKeyGroup(string fileName, TKey[] keys, bool inverseRemove, bool autoReindex)
{
@ -316,7 +343,9 @@ namespace ZeroLevel.Services.PartitionStorage
{
var index = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, _options.FilePartition, _options.KeyComparer);
var offsets = index.GetOffset(keys, true);
using (var reader = GetReadStream(fileName))
if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{
for (int i = 0; i < keys.Length; i++)
{
@ -342,9 +371,12 @@ namespace ZeroLevel.Services.PartitionStorage
}
}
}
}
else
{
using (var reader = GetReadStream(fileName))
if (TryGetReadStream(fileName, out var reader))
{
using (reader)
{
int index = 0;
var keys_arr = keys.OrderBy(k => k).ToArray();
@ -374,6 +406,7 @@ namespace ZeroLevel.Services.PartitionStorage
}
}
}
}
// 2. Temporary file from ranges
var tempPath = Path.GetTempPath();
@ -414,11 +447,21 @@ namespace ZeroLevel.Services.PartitionStorage
}
}
private MemoryStreamReader GetReadStream(string fileName)
private bool TryGetReadStream(string fileName, out MemoryStreamReader reader)
{
try
{
var filePath = Path.Combine(_catalog, fileName);
var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096 * 1024);
return new MemoryStreamReader(stream);
reader = new MemoryStreamReader(stream);
return true;
}
catch (Exception ex)
{
Log.SystemError(ex, "[StorePartitionAccessor.TryGetReadStream]");
}
reader = null;
return false;
}
#endregion

51
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs
Unescape View File

@ -52,11 +52,13 @@ namespace ZeroLevel.Services.PartitionStorage
public void Store(TKey key, TInput value)
{
var fileName = _options.GetFileName(key, _info);
var stream = GetWriteStream(fileName);
if (TryGetWriteStream(fileName, out var stream))
{
_keySerializer.Invoke(stream, key);
Thread.MemoryBarrier();
_inputSerializer.Invoke(stream, value);
}
}
public void CompleteAdding()
{
// Close all write streams
@ -86,18 +88,16 @@ namespace ZeroLevel.Services.PartitionStorage
{
foreach (var file in files)
{
using (var reader = GetReadStream(Path.GetFileName(file)))
if (TryGetReadStream(file, out var reader))
{
using (reader)
{
while (reader.EOS == false)
{
var key = _keyDeserializer.Invoke(reader);
if (reader.EOS)
{
yield return new StorePartitionKeyValueSearchResult<TKey, TInput> { Key = key, Value = default, Found = true };
break;
}
var val = _inputDeserializer.Invoke(reader);
yield return new StorePartitionKeyValueSearchResult<TKey, TInput> { Key = key, Value = val, Found = true };
yield return new StorePartitionKeyValueSearchResult<TKey, TInput> { Key = key, Value = val, Status = SearchResult.Success };
}
}
}
}
@ -115,8 +115,9 @@ namespace ZeroLevel.Services.PartitionStorage
var dict = new Dictionary<TKey, long>();
foreach (var file in files)
{
dict.Clear();
using (var reader = GetReadStream(Path.GetFileName(file)))
if (TryGetReadStream(file, out var reader))
{
using (reader)
{
while (reader.EOS == false)
{
@ -144,6 +145,8 @@ namespace ZeroLevel.Services.PartitionStorage
}
}
}
dict.Clear();
}
}
}
}
@ -185,20 +188,40 @@ namespace ZeroLevel.Services.PartitionStorage
File.Delete(file);
File.Move(tempFile, file, true);
}
private MemoryStreamWriter GetWriteStream(string fileName)
private bool TryGetWriteStream(string fileName, out MemoryStreamWriter writer)
{
try
{
return _writeStreams.GetOrAdd(fileName, k =>
writer = _writeStreams.GetOrAdd(fileName, k =>
{
var filePath = Path.Combine(_catalog, k);
var stream = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.None, 4096 * 1024);
return new MemoryStreamWriter(stream);
});
return true;
}
catch (Exception ex)
{
Log.SystemError(ex, "[StorePartitionBuilder.TryGetWriteStream]");
}
writer = null;
return false;
}
private MemoryStreamReader GetReadStream(string fileName)
private bool TryGetReadStream(string fileName, out MemoryStreamReader reader)
{
try
{
var filePath = Path.Combine(_catalog, fileName);
var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096 * 1024);
return new MemoryStreamReader(stream);
reader = new MemoryStreamReader(stream);
return true;
}
catch (Exception ex)
{
Log.SystemError(ex, "[StorePartitionBuilder.TryGetReadStream]");
}
reader = null;
return false;
}
#endregion

9
ZeroLevel/Services/PartitionStorage/Search/StorePartitionKeyValueSearchResult.cs
Unescape View File

@ -1,8 +1,15 @@
namespace ZeroLevel.Services.PartitionStorage
{
public enum SearchResult
{
Success,
NotFound,
FileLocked
}
public class StorePartitionKeyValueSearchResult<TKey, TValue>
{
public bool Found { get; set; }
public SearchResult Status { get; set; }
public TKey Key { get; set; }
public TValue Value { get; set; }
}

6
ZeroLevel/ZeroLevel.csproj
Unescape View File

@ -6,7 +6,7 @@
</Description>
<Authors>ogoun</Authors>
<Company>ogoun</Company>
<AssemblyVersion>3.3.7.9</AssemblyVersion>
<AssemblyVersion>3.3.8.1</AssemblyVersion>
<PackageReleaseNotes>PartitionStorage optimizations</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2022</Copyright>
@ -14,8 +14,8 @@
<PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<Version>3.3.7.9</Version>
<FileVersion>3.3.7.9</FileVersion>
<Version>3.3.8.1</Version>
<FileVersion>3.3.8.1</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType>

Write Preview
Loading…
Cancel
Save

Powered by TurnKey Linux.