ogoun
/
Zero
mirror of https://github.com/ogoun/Zero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Suppress exception when find invoke in PartitionStorage

Browse Source
pull/4/head
Ogoun 2 years ago
parent 691b40691d
commit f4e014b0e5
9 changed files with 220 additions and 51 deletions
Show Stats Download Patch File Download Diff File

12
PartitionFileStorageTest/Program.cs
Unescape View File

@ -375,8 +375,8 @@ namespace PartitionFileStorageTest
{
counter--;
var pos = reader.Position;
var k = serializer.KeyDeserializer.Invoke(reader);
serializer.ValueDeserializer.Invoke(reader);
serializer.KeyDeserializer.Invoke(reader, out var k);
serializer.ValueDeserializer.Invoke(reader, out var _);
if (counter == 0)
{
index[k] = pos;
@ -392,12 +392,12 @@ namespace PartitionFileStorageTest
var accessor = fileReader.GetAccessor(pair.Value);
using (var reader = new MemoryStreamReader(accessor))
{
var k = serializer.KeyDeserializer.Invoke(reader);
serializer.KeyDeserializer.Invoke(reader, out var k);
if (k != pair.Key)
{
Log.Warning("Broken index");
}
var v = serializer.ValueDeserializer.Invoke(reader);
serializer.ValueDeserializer.Invoke(reader, out var _);
}
}
@ -430,7 +430,7 @@ namespace PartitionFileStorageTest
{
try
{
var key = serializer.KeyDeserializer.Invoke(reader);
serializer.KeyDeserializer.Invoke(reader, out var key);
if (false == dict.ContainsKey(key))
{
dict[key] = new HashSet<ulong>();
@ -439,7 +439,7 @@ namespace PartitionFileStorageTest
{
break;
}
var input = serializer.InputDeserializer.Invoke(reader);
serializer.InputDeserializer.Invoke(reader, out var input);
dict[key].Add(input);
}
catch (Exception ex)

6
ZeroLevel/Services/PartitionStorage/Interfaces/IStoreSerializer.cs
Unescape View File

@ -7,8 +7,8 @@ namespace ZeroLevel.Services.PartitionStorage.Interfaces
{
Action<MemoryStreamWriter, TKey> KeySerializer { get; }
Action<MemoryStreamWriter, TInput> InputSerializer { get; }
Func<MemoryStreamReader, TKey> KeyDeserializer { get; }
Func<MemoryStreamReader, TInput> InputDeserializer { get; }
Func<MemoryStreamReader, TValue> ValueDeserializer { get; }
TryDeserializeMethod<TKey> KeyDeserializer { get; }
TryDeserializeMethod<TInput> InputDeserializer { get; }
TryDeserializeMethod<TValue> ValueDeserializer { get; }
}
}

20
ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs
Unescape View File

@ -60,6 +60,8 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
}
public IEnumerable<StorePartitionKeyValueSearchResult<TKey, TInput>> Iterate()
{
TKey key;
TInput input;
var files = Directory.GetFiles(_catalog);
if (files != null && files.Length > 0)
{
@ -71,9 +73,9 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
{
while (reader.EOS == false)
{
var key = Serializer.KeyDeserializer.Invoke(reader);
var val = Serializer.InputDeserializer.Invoke(reader);
yield return new StorePartitionKeyValueSearchResult<TKey, TInput> { Key = key, Value = val, Status = SearchResult.Success };
if (Serializer.KeyDeserializer.Invoke(reader, out key) == false) break;
if (Serializer.InputDeserializer.Invoke(reader, out input) == false) break;
yield return new StorePartitionKeyValueSearchResult<TKey, TInput> { Key = key, Value = input, Status = SearchResult.Success };
}
}
}
@ -119,12 +121,17 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
internal void CompressFile(string file)
{
TKey key;
TInput input;
var dict = new Dictionary<TKey, HashSet<TInput>>();
using (var reader = new MemoryStreamReader(new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.None, 4096 * 1024)))
{
while (reader.EOS == false)
{
var key = Serializer.KeyDeserializer.Invoke(reader);
if (false == Serializer.KeyDeserializer.Invoke(reader, out key))
{
throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read key.");
}
if (false == dict.ContainsKey(key))
{
dict[key] = new HashSet<TInput>();
@ -133,7 +140,10 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
{
break;
}
var input = Serializer.InputDeserializer.Invoke(reader);
if (false == Serializer.InputDeserializer.Invoke(reader, out input))
{
throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault input value.");
}
dict[key].Add(input);
}
}

76
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs
Unescape View File

@ -31,15 +31,30 @@ namespace ZeroLevel.Services.PartitionStorage
public StorePartitionKeyValueSearchResult<TKey, TValue> Find(TKey key)
{
TKey k;
TValue v;
IViewAccessor memoryAccessor;
if (_options.Index.Enabled)
try
{
var offset = Indexes.GetOffset(key);
memoryAccessor = offset.Length > 0 ? GetViewAccessor(key, offset.Offset, offset.Length) : GetViewAccessor(key, offset.Offset);
if (_options.Index.Enabled)
{
var offset = Indexes.GetOffset(key);
memoryAccessor = offset.Length > 0 ? GetViewAccessor(key, offset.Offset, offset.Length) : GetViewAccessor(key, offset.Offset);
}
else
{
memoryAccessor = GetViewAccessor(key, 0);
}
}
else
catch (Exception ex)
{
memoryAccessor = GetViewAccessor(key, 0);
Log.SystemError(ex, $"[StorePartitionAccessor.Find] Fault get IViewAccessor by key {(key == null ? string.Empty : key.ToString())}");
return new StorePartitionKeyValueSearchResult<TKey, TValue>
{
Key = key,
Status = SearchResult.FileLockedOrUnavaliable,
Value = default
};
}
if (memoryAccessor != null)
{
@ -47,8 +62,8 @@ namespace ZeroLevel.Services.PartitionStorage
{
while (reader.EOS == false)
{
var k = Serializer.KeyDeserializer.Invoke(reader);
var v = Serializer.ValueDeserializer.Invoke(reader);
if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break;
if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break;
var c = _options.KeyComparer(key, k);
if (c == 0) return new StorePartitionKeyValueSearchResult<TKey, TValue>
{
@ -86,6 +101,8 @@ namespace ZeroLevel.Services.PartitionStorage
}
public IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> Iterate()
{
TKey k;
TValue v;
var files = Directory.GetFiles(_catalog);
if (files != null && files.Length > 0)
{
@ -98,8 +115,8 @@ namespace ZeroLevel.Services.PartitionStorage
{
while (reader.EOS == false)
{
var k = Serializer.KeyDeserializer.Invoke(reader);
var v = Serializer.ValueDeserializer.Invoke(reader);
if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break;
if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break;
yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Status = SearchResult.Success };
}
}
@ -109,6 +126,8 @@ namespace ZeroLevel.Services.PartitionStorage
}
public IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> IterateKeyBacket(TKey key)
{
TKey k;
TValue v;
var fileName = _options.GetFileName(key, _info);
var filePath = Path.Combine(_catalog, fileName);
if (File.Exists(filePath))
@ -120,8 +139,8 @@ namespace ZeroLevel.Services.PartitionStorage
{
while (reader.EOS == false)
{
var k = Serializer.KeyDeserializer.Invoke(reader);
var v = Serializer.ValueDeserializer.Invoke(reader);
if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break;
if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break;
yield return new StorePartitionKeyValueSearchResult<TKey, TValue> { Key = k, Value = v, Status = SearchResult.Success };
}
}
@ -181,6 +200,8 @@ namespace ZeroLevel.Services.PartitionStorage
private IEnumerable<StorePartitionKeyValueSearchResult<TKey, TValue>> Find(string fileName,
TKey[] keys)
{
TKey k;
TValue v;
var filePath = Path.Combine(_catalog, fileName);
if (_options.Index.Enabled)
{
@ -204,8 +225,8 @@ namespace ZeroLevel.Services.PartitionStorage
{
while (reader.EOS == false)
{
var k = Serializer.KeyDeserializer.Invoke(reader);
var v = Serializer.ValueDeserializer.Invoke(reader);
if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break;
if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break;
var c = _options.KeyComparer(searchKey, k);
if (c == 0)
{
@ -237,8 +258,8 @@ namespace ZeroLevel.Services.PartitionStorage
var keys_arr = keys.OrderBy(k => k).ToArray();
while (reader.EOS == false && index < keys_arr.Length)
{
var k = Serializer.KeyDeserializer.Invoke(reader);
var v = Serializer.ValueDeserializer.Invoke(reader);
if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break;
if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break;
var c = _options.KeyComparer(keys_arr[index], k);
if (c == 0)
{
@ -269,6 +290,7 @@ namespace ZeroLevel.Services.PartitionStorage
private void RemoveKeyGroup(string fileName, TKey[] keys, bool inverseRemove, bool autoReindex)
{
TKey k;
var filePath = Path.Combine(_catalog, fileName);
// 1. Find ranges
var ranges = new List<FilePositionRange>();
@ -295,8 +317,16 @@ namespace ZeroLevel.Services.PartitionStorage
while (reader.EOS == false)
{
var startPosition = reader.Position;
var k = Serializer.KeyDeserializer.Invoke(reader);
Serializer.ValueDeserializer.Invoke(reader);
if (Serializer.KeyDeserializer.Invoke(reader, out k) == false)
{
Log.Error($"[StorePartitionAccessor.RemoveKeyGroup] Fault remove keys from file '{fileName}'. Incorrect file structure. Fault read key.");
return;
}
if (Serializer.ValueDeserializer.Invoke(reader, out var _) == false)
{
Log.Error($"[StorePartitionAccessor.RemoveKeyGroup] Fault remove keys from file '{fileName}'. Incorrect file structure. Fault read value.");
return;
}
var endPosition = reader.Position;
var c = _options.KeyComparer(searchKey, k);
if (c == 0)
@ -324,8 +354,16 @@ namespace ZeroLevel.Services.PartitionStorage
while (reader.EOS == false && index < keys_arr.Length)
{
var startPosition = reader.Position;
var k = Serializer.KeyDeserializer.Invoke(reader);
Serializer.ValueDeserializer.Invoke(reader);
if (Serializer.KeyDeserializer.Invoke(reader, out k) == false)
{
Log.Error($"[StorePartitionAccessor.RemoveKeyGroup] Fault remove keys from file '{fileName}'. Incorrect file structure. Fault read key.");
return;
}
if (Serializer.ValueDeserializer.Invoke(reader, out var _) == false)
{
Log.Error($"[StorePartitionAccessor.RemoveKeyGroup] Fault remove keys from file '{fileName}'. Incorrect file structure. Fault read value.");
return;
}
var endPosition = reader.Position;
var c = _options.KeyComparer(keys_arr[index], k);
if (c == 0)

18
ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs
Unescape View File

@ -63,6 +63,8 @@ namespace ZeroLevel.Services.PartitionStorage
}
public IEnumerable<StorePartitionKeyValueSearchResult<TKey, TInput>> Iterate()
{
TKey key;
TInput val;
var files = Directory.GetFiles(_catalog);
if (files != null && files.Length > 0)
{
@ -75,8 +77,8 @@ namespace ZeroLevel.Services.PartitionStorage
{
while (reader.EOS == false)
{
var key = Serializer.KeyDeserializer.Invoke(reader);
var val = Serializer.InputDeserializer.Invoke(reader);
if (Serializer.KeyDeserializer.Invoke(reader, out key) == false) break;
if (Serializer.InputDeserializer.Invoke(reader, out val) == false) break;
yield return new StorePartitionKeyValueSearchResult<TKey, TInput> { Key = key, Value = val, Status = SearchResult.Success };
}
}
@ -135,6 +137,8 @@ namespace ZeroLevel.Services.PartitionStorage
internal void CompressFile(string file)
{
TKey key;
TInput input;
var dict = new Dictionary<TKey, HashSet<TInput>>();
var accessor = PhisicalFileAccessorCachee.GetDataAccessor(file, 0);
if (accessor != null)
@ -143,7 +147,10 @@ namespace ZeroLevel.Services.PartitionStorage
{
while (reader.EOS == false)
{
var key = Serializer.KeyDeserializer.Invoke(reader);
if (Serializer.KeyDeserializer.Invoke(reader, out key) == false)
{
throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read key.");
}
if (false == dict.ContainsKey(key))
{
dict[key] = new HashSet<TInput>();
@ -152,7 +159,10 @@ namespace ZeroLevel.Services.PartitionStorage
{
break;
}
var input = Serializer.InputDeserializer.Invoke(reader);
if (Serializer.InputDeserializer.Invoke(reader, out input) == false)
{
throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read input value.");
}
dict[key].Add(input);
}
}

21
ZeroLevel/Services/PartitionStorage/StoreStandartSerializer.cs
Unescape View File

@ -4,33 +4,36 @@ using ZeroLevel.Services.Serialization;
namespace ZeroLevel.Services.PartitionStorage
{
// TODO INTERNAL
public sealed class StoreStandartSerializer<TKey, TInput, TValue>
: IStoreSerializer<TKey, TInput, TValue>
{
private readonly Action<MemoryStreamWriter, TKey> _keySerializer;
private readonly Action<MemoryStreamWriter, TInput> _inputSerializer;
private readonly Func<MemoryStreamReader, TKey> _keyDeserializer;
private readonly Func<MemoryStreamReader, TInput> _inputDeserializer;
private readonly Func<MemoryStreamReader, TValue> _valueDeserializer;
private readonly TryDeserializeMethod<TKey> _keyDeserializer;
private readonly TryDeserializeMethod<TInput> _inputDeserializer;
private readonly TryDeserializeMethod<TValue> _valueDeserializer;
public StoreStandartSerializer()
{
_keySerializer = MessageSerializer.GetSerializer<TKey>();
_inputSerializer = MessageSerializer.GetSerializer<TInput>();
_keyDeserializer = MessageSerializer.GetDeserializer<TKey>();
_inputDeserializer = MessageSerializer.GetDeserializer<TInput>();
_valueDeserializer = MessageSerializer.GetDeserializer<TValue>();
_keyDeserializer = MessageSerializer.GetSafetyDeserializer<TKey>();
_inputDeserializer = MessageSerializer.GetSafetyDeserializer<TInput>();
_valueDeserializer = MessageSerializer.GetSafetyDeserializer<TValue>();
}
public Action<MemoryStreamWriter, TKey> KeySerializer => _keySerializer;
public Action<MemoryStreamWriter, TInput> InputSerializer => _inputSerializer;
public Func<MemoryStreamReader, TKey> KeyDeserializer => _keyDeserializer;
public TryDeserializeMethod<TKey> KeyDeserializer => _keyDeserializer;
public Func<MemoryStreamReader, TInput> InputDeserializer => _inputDeserializer;
public TryDeserializeMethod<TInput> InputDeserializer => _inputDeserializer;
public Func<MemoryStreamReader, TValue> ValueDeserializer => _valueDeserializer;
public TryDeserializeMethod<TValue> ValueDeserializer => _valueDeserializer;
}
}

67
ZeroLevel/Services/Serialization/MemoryStreamReader.cs
Unescape View File

@ -3,6 +3,7 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Runtime.Serialization;
using System.Text;
using ZeroLevel.Services.Extensions;
using ZeroLevel.Services.Memory;
@ -203,6 +204,36 @@ namespace ZeroLevel.Services.Serialization
return buffer;
}
public bool TryReadBuffer(int count, out byte[] buffer)
{
if (CheckOutOfRange(count))
{
buffer = null;
return false;
}
try
{
buffer = _accessor.ReadBuffer(count);
if (_reverseByteOrder && count > 1)
{
byte b;
for (int i = 0; i < (count >> 1); i++)
{
b = buffer[i];
buffer[i] = buffer[count - i - 1];
buffer[count - i - 1] = b;
}
}
}
catch (Exception ex)
{
Log.SystemError(ex, $"[MemoryStreamReader.TryReadBuffer] Fault read {count} bytes");
buffer = null;
return false;
}
return true;
}
/// <summary>
/// Reading the datetime
/// </summary>
@ -1121,6 +1152,42 @@ namespace ZeroLevel.Services.Serialization
return item;
}
public bool TryReadByte(out byte b)
{
if (TryReadBuffer(1, out var buffer))
{
b = buffer[0];
return true;
}
b = default;
return false;
}
public bool TryRead<T>(out T item) where T : IBinarySerializable
{
if (TryReadByte(out var type))
{
if (type == 0)
{
item = default(T);
return true;
}
try
{
var o = (IBinarySerializable)FormatterServices.GetUninitializedObject(typeof(T));
o.Deserialize(this);
item = (T)o;
return true;
}
catch (Exception ex)
{
Log.SystemError(ex, "[MemoryStreamReader.TryRead]");
}
}
item = default;
return false;
}
public T Read<T>(object arg) where T : IBinarySerializable
{
byte type = ReadByte();

43
ZeroLevel/Services/Serialization/MessageSerializer.cs
Unescape View File

@ -1,11 +1,11 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Runtime.Serialization;
namespace ZeroLevel.Services.Serialization
{
public delegate bool TryDeserializeMethod<T>(MemoryStreamReader reader, out T output);
public static class MessageSerializer
{
public static byte[] Serialize<T>(T obj)
@ -54,6 +54,47 @@ namespace ZeroLevel.Services.Serialization
return (r) => PrimitiveTypeSerializer.Deserialize<T>(r);
}
static bool TryObjectDeserialize<T>(MemoryStreamReader reader, out T output)
{
try
{
var o = (IBinarySerializable)FormatterServices.GetUninitializedObject(typeof(T));
o.Deserialize(reader);
output = (T)o;
return true;
}
catch (Exception ex)
{
Log.SystemError(ex, $"[MessageSerializer.TryObjectDeserialize] Fault deserialize type {typeof(T).Name}");
output = default;
}
return false;
}
static bool TryPrimitiveTypeDeserialize<T>(MemoryStreamReader reader, out T output)
{
try
{
output = PrimitiveTypeSerializer.Deserialize<T>(reader);
return true;
}
catch (Exception ex)
{
Log.SystemError(ex, $"[MessageSerializer.TryPrimitiveTypeDeserialize] Fault deserialize type {typeof(T).Name}");
output = default;
}
return false;
}
public static TryDeserializeMethod<T> GetSafetyDeserializer<T>()
{
if (typeof(IBinarySerializable).IsAssignableFrom(typeof(T)))
{
return TryObjectDeserialize<T>;
}
return TryPrimitiveTypeDeserialize<T>;
}
public static byte[] SerializeCompatible(object obj)
{
if (null == obj)

8
ZeroLevel/ZeroLevel.csproj
Unescape View File

@ -6,16 +6,16 @@
</Description>
<Authors>ogoun</Authors>
<Company>ogoun</Company>
<AssemblyVersion>3.3.8.8</AssemblyVersion>
<PackageReleaseNotes>MMF for partition storage</PackageReleaseNotes>
<AssemblyVersion>3.3.8.9</AssemblyVersion>
<PackageReleaseNotes>Partition storage. Suppress exception when find invoke</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2023</Copyright>
<PackageLicenseUrl></PackageLicenseUrl>
<PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<Version>3.3.8.8</Version>
<FileVersion>3.3.8.8</FileVersion>
<Version>3.3.8.9</Version>
<FileVersion>3.3.8.9</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType>

Write Preview
Loading…
Cancel
Save

Powered by TurnKey Linux.