ogoun
/
Zero
mirror of https://github.com/ogoun/Zero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PartitionStorage. Fix bugs of multithreads

Browse Source
pull/4/head
Ogoun 2 years ago
parent 5cf967fc61
commit 0d7e23320e
13 changed files with 238 additions and 45 deletions
Show Stats Download Patch File Download Diff File

66
PartitionFileStorageTest/Program.cs
Unescape View File

@ -4,13 +4,14 @@ using System.Text;
using ZeroLevel; using ZeroLevel;
using ZeroLevel.Services.FileSystem; using ZeroLevel.Services.FileSystem;
using ZeroLevel.Services.PartitionStorage; using ZeroLevel.Services.PartitionStorage;
using ZeroLevel.Services.Serialization;
namespace PartitionFileStorageTest namespace PartitionFileStorageTest
{ {
internal class Program internal class Program
{ {
// const int PAIRS_COUNT = 200_000_000; // const int PAIRS_COUNT = 200_000_000;
const int PAIRS_COUNT = 200_000; const int PAIRS_COUNT = 2000_000;
private class Metadata private class Metadata
{ {
@ -241,6 +242,11 @@ namespace PartitionFileStorageTest
} }
}); });
if (storePart.TotalRecords != insertCount)
{
Log.Error($"Count of stored record no equals expected. Recorded: {storePart.TotalRecords}. Expected: {insertCount}");
}
sw.Stop(); sw.Stop();
Log.Info($"Fill journal: {sw.ElapsedMilliseconds}ms"); Log.Info($"Fill journal: {sw.ElapsedMilliseconds}ms");
sw.Restart(); sw.Restart();
@ -264,6 +270,11 @@ namespace PartitionFileStorageTest
merger.Store(key, val); merger.Store(key, val);
}); });
if (merger.TotalRecords != (pairs.Count - insertCount))
{
Log.Error($"Count of stored record no equals expected. Recorded: {merger.TotalRecords}. Expected: {(pairs.Count - insertCount)}");
}
Log.Info($"Merge journal filled: {sw.ElapsedMilliseconds}ms. Total data count: {pairs.Count}"); Log.Info($"Merge journal filled: {sw.ElapsedMilliseconds}ms. Total data count: {pairs.Count}");
merger.Compress(); // auto reindex merger.Compress(); // auto reindex
sw.Stop(); sw.Stop();
@ -334,8 +345,49 @@ namespace PartitionFileStorageTest
Log.Info("Completed"); Log.Info("Completed");
} }
private static void FaultIndexTest(string filePath)
{
var serializer = new StoreStandartSerializer<ulong, ulong, byte[]>();
// 1 build index
var index = new Dictionary<ulong, long>();
using (var reader = new MemoryStreamReader(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096 * 1024)))
{
var counter = 1;
while (reader.EOS == false)
{
counter--;
var pos = reader.Position;
var k = serializer.KeyDeserializer.Invoke(reader);
serializer.ValueDeserializer.Invoke(reader);
if (counter == 0)
{
index[k] = pos;
counter = 16;
}
}
}
// 2 Test index
using (var reader = new MemoryStreamReader(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096 * 1024)))
{
foreach (var pair in index)
{
reader.Stream.Seek(pair.Value, SeekOrigin.Begin);
var k = serializer.KeyDeserializer.Invoke(reader);
if (k != pair.Key)
{
Log.Warning("Broken index");
}
var v = serializer.ValueDeserializer.Invoke(reader);
}
}
}
static void Main(string[] args) static void Main(string[] args)
{ {
/*FaultIndexTest(@"H:\temp\85");
return;*/
var root = @"H:\temp"; var root = @"H:\temp";
var options = new StoreOptions<ulong, ulong, byte[], Metadata> var options = new StoreOptions<ulong, ulong, byte[], Metadata>
{ {
@ -343,7 +395,8 @@ namespace PartitionFileStorageTest
{ {
Enabled = true, Enabled = true,
StepType = IndexStepType.Step, StepType = IndexStepType.Step,
StepValue = 16 StepValue = 16,
EnableIndexInMemoryCachee= true
}, },
RootFolder = root, RootFolder = root,
FilePartition = new StoreFilePartition<ulong, Metadata>("Last three digits", (ctn, date) => (ctn % 128).ToString()), FilePartition = new StoreFilePartition<ulong, Metadata>("Last three digits", (ctn, date) => (ctn % 128).ToString()),
@ -365,7 +418,8 @@ namespace PartitionFileStorageTest
{ {
Enabled = true, Enabled = true,
StepType = IndexStepType.Step, StepType = IndexStepType.Step,
StepValue = 16 StepValue = 16,
EnableIndexInMemoryCachee = true
}, },
RootFolder = root, RootFolder = root,
FilePartition = new StoreFilePartition<ulong, Metadata>("Last three digits", (ctn, date) => (ctn % 128).ToString()), FilePartition = new StoreFilePartition<ulong, Metadata>("Last three digits", (ctn, date) => (ctn % 128).ToString()),
@ -393,10 +447,14 @@ namespace PartitionFileStorageTest
// FastTest(options); // FastTest(options);
FSUtils.CleanAndTestFolder(root); FSUtils.CleanAndTestFolder(root);
FullStoreMultithreadTest(optionsMultiThread, pairs);
FullStoreTest(options, pairs); FullStoreTest(options, pairs);
/* /*
FSUtils.CleanAndTestFolder(root); FSUtils.CleanAndTestFolder(root);
FullStoreMultithreadTest(optionsMultiThread, pairs);
*/ */
Console.ReadKey(); Console.ReadKey();
} }

25
ZeroLevel.NN/Architectures/YoloV5/Yolov5Detector.cs
Unescape View File

@ -9,8 +9,8 @@ namespace ZeroLevel.NN.Architectures.YoloV5
{ {
private int INPUT_WIDTH = 640; private int INPUT_WIDTH = 640;
private int INPUT_HEIGHT = 640; private int INPUT_HEIGHT = 640;
private int CROP_WIDTH = 1440; private int CROP_WIDTH = 1280;
private int CROP_HEIGHT = 1440; private int CROP_HEIGHT = 1280;
public Yolov5Detector(string modelPath, int inputWidth = 640, int inputHeight = 640, bool gpu = false) public Yolov5Detector(string modelPath, int inputWidth = 640, int inputHeight = 640, bool gpu = false)
: base(modelPath, gpu) : base(modelPath, gpu)
@ -43,7 +43,15 @@ namespace ZeroLevel.NN.Architectures.YoloV5
var result = new List<YoloPrediction>(); var result = new List<YoloPrediction>();
Extract(new Dictionary<string, Tensor<float>> { { "images", input } }, d => Extract(new Dictionary<string, Tensor<float>> { { "images", input } }, d =>
{ {
var output = d["output"]; Tensor<float> output;
if (d.ContainsKey("output"))
{
output = d["output"];
}
else
{
output = d.First().Value;
}
/* /*
var output350 = d["350"]; var output350 = d["350"];
var output498 = d["498"]; var output498 = d["498"];
@ -93,7 +101,16 @@ namespace ZeroLevel.NN.Architectures.YoloV5
{ {
Extract(new Dictionary<string, Tensor<float>> { { "images", input.Tensor } }, d => Extract(new Dictionary<string, Tensor<float>> { { "images", input.Tensor } }, d =>
{ {
var output = d["output"]; Tensor<float> output;
if (d.ContainsKey("output"))
{
output = d["output"];
}
else
{
output = d.First().Value;
}
/* /*
var output350 = d["350"]; var output350 = d["350"];
var output498 = d["498"]; var output498 = d["498"];

7
ZeroLevel.NN/Services/ImagePreprocessor.cs
Unescape View File

@ -143,6 +143,13 @@ namespace ZeroLevel.NN
{ {
var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction; var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction;
if (image.PixelType.BitsPerPixel != 24)
{
var i = image;
image = i.CloneAs<Rgb24>();
i.Dispose();
}
((Image<Rgb24>)image).ProcessPixelRows(pixels => ((Image<Rgb24>)image).ProcessPixelRows(pixels =>
{ {
if (options.InvertXY) if (options.InvertXY)

2
ZeroLevel.NN/ZeroLevel.NN.csproj
Unescape View File

@ -35,7 +35,7 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Aurigma.GraphicsMill.Core.x64" Version="11.0.11" /> <PackageReference Include="Aurigma.GraphicsMill.Core.x64" Version="11.0.27" />
<PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.13.1" /> <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.13.1" />
<PackageReference Include="SixLabors.ImageSharp" Version="2.1.3" /> <PackageReference Include="SixLabors.ImageSharp" Version="2.1.3" />
<PackageReference Include="SixLabors.ImageSharp.Drawing" Version="1.0.0-beta14" /> <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="1.0.0-beta14" />

69
ZeroLevel/Services/PartitionStorage/Indexes/KeysIndex/StorePartitionSparseIndex.cs
Unescape View File

@ -8,18 +8,20 @@ namespace ZeroLevel.Services.PartitionStorage
internal sealed class StorePartitionSparseIndex<TKey, TMeta> internal sealed class StorePartitionSparseIndex<TKey, TMeta>
: IStorePartitionIndex<TKey> : IStorePartitionIndex<TKey>
{ {
private readonly Dictionary<string, KeyIndex<TKey>[]> _indexCachee
= new Dictionary<string, KeyIndex<TKey>[]>(1024);
private readonly StoreFilePartition<TKey, TMeta> _filePartition; private readonly StoreFilePartition<TKey, TMeta> _filePartition;
private readonly Func<TKey, TKey, int> _keyComparer; private readonly Func<TKey, TKey, int> _keyComparer;
private readonly string _indexFolder; private readonly string _indexFolder;
private readonly bool _indexExists = false; private readonly bool _indexExists = false;
private readonly bool _enableIndexInMemoryCachee;
private readonly Func<MemoryStreamReader, TKey> _keyDeserializer; private readonly Func<MemoryStreamReader, TKey> _keyDeserializer;
private readonly TMeta _meta; private readonly TMeta _meta;
private readonly Dictionary<string, KeyIndex<TKey>[]> _indexCachee = null;
public StorePartitionSparseIndex(string partitionFolder, TMeta meta, public StorePartitionSparseIndex(string partitionFolder, TMeta meta,
StoreFilePartition<TKey, TMeta> filePartition, StoreFilePartition<TKey, TMeta> filePartition,
Func<TKey, TKey, int> keyComparer) Func<TKey, TKey, int> keyComparer,
bool enableIndexInMemoryCachee)
{ {
_indexFolder = Path.Combine(partitionFolder, "__indexes__"); _indexFolder = Path.Combine(partitionFolder, "__indexes__");
_indexExists = Directory.Exists(_indexFolder); _indexExists = Directory.Exists(_indexFolder);
@ -27,6 +29,11 @@ namespace ZeroLevel.Services.PartitionStorage
_keyComparer = keyComparer; _keyComparer = keyComparer;
_filePartition = filePartition; _filePartition = filePartition;
_keyDeserializer = MessageSerializer.GetDeserializer<TKey>(); _keyDeserializer = MessageSerializer.GetDeserializer<TKey>();
_enableIndexInMemoryCachee = enableIndexInMemoryCachee;
if (_enableIndexInMemoryCachee)
{
_indexCachee = new Dictionary<string, KeyIndex<TKey>[]>(1024);
}
} }
public KeyIndex<TKey> GetOffset(TKey key) public KeyIndex<TKey> GetOffset(TKey key)
@ -99,11 +106,60 @@ namespace ZeroLevel.Services.PartitionStorage
return index[position]; return index[position];
} }
public void ResetCachee()
{
if (_enableIndexInMemoryCachee)
{
lock (_casheupdatelock)
{
_indexCachee.Clear();
}
}
}
public void RemoveCacheeItem(string name)
{
if (_enableIndexInMemoryCachee)
{
lock (_casheupdatelock)
{
_indexCachee.Remove(name);
}
}
}
private readonly object _casheupdatelock = new object();
private KeyIndex<TKey>[] GetFileIndex(TKey key) private KeyIndex<TKey>[] GetFileIndex(TKey key)
{ {
var indexName = _filePartition.FileNameExtractor.Invoke(key, _meta); var indexName = _filePartition.FileNameExtractor.Invoke(key, _meta);
if (_indexCachee.TryGetValue(indexName, out var index)) return index; try
{
if (_enableIndexInMemoryCachee)
{
if (_indexCachee.TryGetValue(indexName, out var index))
{
return index;
}
lock (_casheupdatelock)
{
_indexCachee[indexName] = ReadIndexesFromIndexFile(indexName);
return _indexCachee[indexName];
}
}
else
{
return ReadIndexesFromIndexFile(indexName);
}
}
catch (Exception ex)
{
Log.SystemError(ex, "[StorePartitionSparseIndex.GetFileIndex] No cachee");
}
return null;
}
private KeyIndex<TKey>[] ReadIndexesFromIndexFile(string indexName)
{
var file = Path.Combine(_indexFolder, indexName); var file = Path.Combine(_indexFolder, indexName);
if (File.Exists(file)) if (File.Exists(file))
{ {
@ -117,8 +173,7 @@ namespace ZeroLevel.Services.PartitionStorage
list.Add(new KeyIndex<TKey> { Key = k, Offset = o }); list.Add(new KeyIndex<TKey> { Key = k, Offset = o });
} }
} }
_indexCachee[indexName] = list.ToArray(); return list.ToArray();
return _indexCachee[indexName];
} }
return null; return null;
} }

1
ZeroLevel/Services/PartitionStorage/Options/IndexOptions.cs
Unescape View File

@ -11,5 +11,6 @@
public bool Enabled { get; set; } public bool Enabled { get; set; }
public IndexStepType StepType { get; set; } = IndexStepType.AbsoluteCount; public IndexStepType StepType { get; set; } = IndexStepType.AbsoluteCount;
public int StepValue { get; set; } = 64; public int StepValue { get; set; } = 64;
public bool EnableIndexInMemoryCachee { get; set; } = false;
} }
} }

32
ZeroLevel/Services/PartitionStorage/Partition/BasePartition.cs
Unescape View File

@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Concurrent; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Threading;
using ZeroLevel.Services.FileSystem; using ZeroLevel.Services.FileSystem;
using ZeroLevel.Services.PartitionStorage.Interfaces; using ZeroLevel.Services.PartitionStorage.Interfaces;
using ZeroLevel.Services.Serialization; using ZeroLevel.Services.Serialization;
@ -21,7 +22,7 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
protected readonly StoreOptions<TKey, TInput, TValue, TMeta> _options; protected readonly StoreOptions<TKey, TInput, TValue, TMeta> _options;
private readonly IndexBuilder<TKey, TValue> _indexBuilder; private readonly IndexBuilder<TKey, TValue> _indexBuilder;
private readonly ConcurrentDictionary<string, MemoryStreamWriter> _writeStreams = new ConcurrentDictionary<string, MemoryStreamWriter>(); private readonly Dictionary<string, MemoryStreamWriter> _writeStreams = new Dictionary<string, MemoryStreamWriter>();
internal BasePartition(StoreOptions<TKey, TInput, TValue, TMeta> options, internal BasePartition(StoreOptions<TKey, TInput, TValue, TMeta> options,
TMeta info, TMeta info,
@ -101,14 +102,33 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
{ {
try try
{ {
writer = _writeStreams.GetOrAdd(fileName, k => bool taken = false;
Monitor.Enter(_writeStreams, ref taken);
try
{
if (_writeStreams.TryGetValue(fileName, out var w))
{
writer = w;
return true;
}
else
{ {
var filePath = Path.Combine(_catalog, k); var filePath = Path.Combine(_catalog, fileName);
var stream = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.None, 4096 * 1024); var stream = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.None, 4096 * 1024);
return new MemoryStreamWriter(stream); var new_w = new MemoryStreamWriter(stream);
}); _writeStreams[fileName] = new_w;
writer = new_w;
return true; return true;
} }
}
finally
{
if (taken)
{
Monitor.Exit(_writeStreams);
}
}
}
catch (Exception ex) catch (Exception ex)
{ {
Log.SystemError(ex, "[StorePartitionBuilder.TryGetWriteStream]"); Log.SystemError(ex, "[StorePartitionBuilder.TryGetWriteStream]");

4
ZeroLevel/Services/PartitionStorage/Partition/StoreMergePartitionAccessor.cs
Unescape View File

@ -144,6 +144,10 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
else
{
Log.SystemError($"File not found '{filePath}'");
}
} }
#endregion #endregion

32
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs
Unescape View File

@ -11,12 +11,18 @@ namespace ZeroLevel.Services.PartitionStorage
internal sealed class StorePartitionAccessor<TKey, TInput, TValue, TMeta> internal sealed class StorePartitionAccessor<TKey, TInput, TValue, TMeta>
: BasePartition<TKey, TInput, TValue, TMeta>, IStorePartitionAccessor<TKey, TInput, TValue> : BasePartition<TKey, TInput, TValue, TMeta>, IStorePartitionAccessor<TKey, TInput, TValue>
{ {
private readonly StorePartitionSparseIndex<TKey, TMeta> Indexes;
public StorePartitionAccessor(StoreOptions<TKey, TInput, TValue, TMeta> options, public StorePartitionAccessor(StoreOptions<TKey, TInput, TValue, TMeta> options,
TMeta info, TMeta info,
IStoreSerializer<TKey, TInput, TValue> serializer) IStoreSerializer<TKey, TInput, TValue> serializer)
: base(options, info, serializer) : base(options, info, serializer)
{ {
if (options == null) throw new ArgumentNullException(nameof(options)); if (options == null) throw new ArgumentNullException(nameof(options));
if (options.Index.Enabled)
{
Indexes = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, options.FilePartition, options.KeyComparer, options.Index.EnableIndexInMemoryCachee);
}
} }
#region IStorePartitionAccessor #region IStorePartitionAccessor
@ -28,8 +34,7 @@ namespace ZeroLevel.Services.PartitionStorage
long startOffset = 0; long startOffset = 0;
if (_options.Index.Enabled) if (_options.Index.Enabled)
{ {
var index = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, _options.FilePartition, _options.KeyComparer); var offset = Indexes.GetOffset(key);
var offset = index.GetOffset(key);
startOffset = offset.Offset; startOffset = offset.Offset;
} }
if (TryGetReadStream(fileName, out var reader)) if (TryGetReadStream(fileName, out var reader))
@ -130,7 +135,14 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
public void RebuildIndex() => RebuildIndexes(); public void RebuildIndex()
{
RebuildIndexes();
if (_options.Index.Enabled)
{
Indexes.ResetCachee();
}
}
public void RemoveAllExceptKey(TKey key, bool autoReindex = true) public void RemoveAllExceptKey(TKey key, bool autoReindex = true)
{ {
RemoveAllExceptKeys(new[] { key }, autoReindex); RemoveAllExceptKeys(new[] { key }, autoReindex);
@ -144,6 +156,10 @@ namespace ZeroLevel.Services.PartitionStorage
foreach (var group in results) foreach (var group in results)
{ {
RemoveKeyGroup(group.FileName, group.Keys, false, autoReindex); RemoveKeyGroup(group.FileName, group.Keys, false, autoReindex);
if (_options.Index.Enabled)
{
Indexes.RemoveCacheeItem(group.FileName);
}
} }
} }
public void RemoveKey(TKey key, bool autoReindex = false) public void RemoveKey(TKey key, bool autoReindex = false)
@ -159,6 +175,10 @@ namespace ZeroLevel.Services.PartitionStorage
foreach (var group in results) foreach (var group in results)
{ {
RemoveKeyGroup(group.FileName, group.Keys, true, autoReindex); RemoveKeyGroup(group.FileName, group.Keys, true, autoReindex);
if (_options.Index.Enabled)
{
Indexes.RemoveCacheeItem(group.FileName);
}
} }
} }
#endregion #endregion
@ -172,8 +192,7 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
if (_options.Index.Enabled) if (_options.Index.Enabled)
{ {
var index = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, _options.FilePartition, _options.KeyComparer); var offsets = Indexes.GetOffset(keys, true);
var offsets = index.GetOffset(keys, true);
if (TryGetReadStream(fileName, out var reader)) if (TryGetReadStream(fileName, out var reader))
{ {
using (reader) using (reader)
@ -257,8 +276,7 @@ namespace ZeroLevel.Services.PartitionStorage
var ranges = new List<FilePositionRange>(); var ranges = new List<FilePositionRange>();
if (_options.Index.Enabled && autoReindex) if (_options.Index.Enabled && autoReindex)
{ {
var index = new StorePartitionSparseIndex<TKey, TMeta>(_catalog, _info, _options.FilePartition, _options.KeyComparer); var offsets = Indexes.GetOffset(keys, true);
var offsets = index.GetOffset(keys, true);
if (TryGetReadStream(fileName, out var reader)) if (TryGetReadStream(fileName, out var reader))
{ {
using (reader) using (reader)

23
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs
Unescape View File

@ -14,7 +14,7 @@ namespace ZeroLevel.Services.PartitionStorage
internal sealed class StorePartitionBuilder<TKey, TInput, TValue, TMeta> internal sealed class StorePartitionBuilder<TKey, TInput, TValue, TMeta>
: BasePartition<TKey, TInput, TValue, TMeta>, IStorePartitionBuilder<TKey, TInput, TValue> : BasePartition<TKey, TInput, TValue, TMeta>, IStorePartitionBuilder<TKey, TInput, TValue>
{ {
private readonly Action<TKey, TInput> _storeMethod; private readonly Func<TKey, TInput, bool> _storeMethod;
private long _totalRecords = 0; private long _totalRecords = 0;
@ -41,9 +41,11 @@ namespace ZeroLevel.Services.PartitionStorage
public void Store(TKey key, TInput value) public void Store(TKey key, TInput value)
{ {
_storeMethod.Invoke(key, value); if (_storeMethod.Invoke(key, value))
{
Interlocked.Increment(ref _totalRecords); Interlocked.Increment(ref _totalRecords);
} }
}
public void CompleteAdding() public void CompleteAdding()
{ {
@ -84,7 +86,7 @@ namespace ZeroLevel.Services.PartitionStorage
#endregion #endregion
#region Private methods #region Private methods
private void StoreDirect(TKey key, TInput value) private bool StoreDirect(TKey key, TInput value)
{ {
var groupKey = _options.GetFileName(key, _info); var groupKey = _options.GetFileName(key, _info);
if (TryGetWriteStream(groupKey, out var stream)) if (TryGetWriteStream(groupKey, out var stream))
@ -92,9 +94,15 @@ namespace ZeroLevel.Services.PartitionStorage
Serializer.KeySerializer.Invoke(stream, key); Serializer.KeySerializer.Invoke(stream, key);
Thread.MemoryBarrier(); Thread.MemoryBarrier();
Serializer.InputSerializer.Invoke(stream, value); Serializer.InputSerializer.Invoke(stream, value);
return true;
}
else
{
Log.SystemError($"Fault create write stream for key '{groupKey}'");
} }
return false;
} }
private void StoreDirectSafe(TKey key, TInput value) private bool StoreDirectSafe(TKey key, TInput value)
{ {
var groupKey = _options.GetFileName(key, _info); var groupKey = _options.GetFileName(key, _info);
bool lockTaken = false; bool lockTaken = false;
@ -106,6 +114,7 @@ namespace ZeroLevel.Services.PartitionStorage
Serializer.KeySerializer.Invoke(stream, key); Serializer.KeySerializer.Invoke(stream, key);
Thread.MemoryBarrier(); Thread.MemoryBarrier();
Serializer.InputSerializer.Invoke(stream, value); Serializer.InputSerializer.Invoke(stream, value);
return true;
} }
finally finally
{ {
@ -115,6 +124,11 @@ namespace ZeroLevel.Services.PartitionStorage
} }
} }
} }
else
{
Log.SystemError($"Fault create write stream for key '{groupKey}'");
}
return false;
} }
internal void CompressFile(string file) internal void CompressFile(string file)
@ -145,6 +159,7 @@ namespace ZeroLevel.Services.PartitionStorage
{ {
var v = _options.MergeFunction(pair.Value); var v = _options.MergeFunction(pair.Value);
writer.SerializeCompatible(pair.Key); writer.SerializeCompatible(pair.Key);
Thread.MemoryBarrier();
writer.SerializeCompatible(v); writer.SerializeCompatible(v);
} }
} }

3
ZeroLevel/Services/PartitionStorage/StoreStandartSerializer.cs
Unescape View File

@ -4,7 +4,8 @@ using ZeroLevel.Services.Serialization;
namespace ZeroLevel.Services.PartitionStorage namespace ZeroLevel.Services.PartitionStorage
{ {
internal sealed class StoreStandartSerializer<TKey, TInput, TValue> // TODO INTERNAL
public sealed class StoreStandartSerializer<TKey, TInput, TValue>
: IStoreSerializer<TKey, TInput, TValue> : IStoreSerializer<TKey, TInput, TValue>
{ {
private readonly Action<MemoryStreamWriter, TKey> _keySerializer; private readonly Action<MemoryStreamWriter, TKey> _keySerializer;

3
ZeroLevel/Services/Reflection/TypeHelpers.cs
Unescape View File

@ -47,10 +47,7 @@ namespace ZeroLevel.Services.Reflection
return false; return false;
} }
} }
<<<<<<< Updated upstream
=======
>>>>>>> Stashed changes
public static bool IsNumericTypeWithFloating(Type type) public static bool IsNumericTypeWithFloating(Type type)
{ {
switch (Type.GetTypeCode(type)) switch (Type.GetTypeCode(type))

2
ZeroLevel/Services/Serialization/MemoryStreamReader.cs
Unescape View File

@ -262,7 +262,7 @@ namespace ZeroLevel.Services.Serialization
/// </summary> /// </summary>
public bool CheckOutOfRange(int offset) public bool CheckOutOfRange(int offset)
{ {
return (_stream.Position + offset) > _stream.Length; return offset < 0 || (_stream.Position + offset) > _stream.Length;
} }
#region Extensions #region Extensions

Write Preview
Loading…
Cancel
Save

Powered by TurnKey Linux.