From f4e014b0e5757ba2099b28dfc32456aa016d2da0 Mon Sep 17 00:00:00 2001 From: Ogoun Date: Fri, 20 Jan 2023 18:05:48 +0300 Subject: [PATCH] Suppress exception when find invoke in PartitionStorage --- PartitionFileStorageTest/Program.cs | 12 +-- .../Interfaces/IStoreSerializer.cs | 6 +- .../CompactKeyStorePartitionBuilder.cs | 20 +++-- .../Partition/StorePartitionAccessor.cs | 76 ++++++++++++++----- .../Partition/StorePartitionBuilder.cs | 18 ++++- .../StoreStandartSerializer.cs | 21 ++--- .../Serialization/MemoryStreamReader.cs | 67 ++++++++++++++++ .../Serialization/MessageSerializer.cs | 43 ++++++++++- ZeroLevel/ZeroLevel.csproj | 8 +- 9 files changed, 220 insertions(+), 51 deletions(-) diff --git a/PartitionFileStorageTest/Program.cs b/PartitionFileStorageTest/Program.cs index b845d63..8bd0fa7 100644 --- a/PartitionFileStorageTest/Program.cs +++ b/PartitionFileStorageTest/Program.cs @@ -375,8 +375,8 @@ namespace PartitionFileStorageTest { counter--; var pos = reader.Position; - var k = serializer.KeyDeserializer.Invoke(reader); - serializer.ValueDeserializer.Invoke(reader); + serializer.KeyDeserializer.Invoke(reader, out var k); + serializer.ValueDeserializer.Invoke(reader, out var _); if (counter == 0) { index[k] = pos; @@ -392,12 +392,12 @@ namespace PartitionFileStorageTest var accessor = fileReader.GetAccessor(pair.Value); using (var reader = new MemoryStreamReader(accessor)) { - var k = serializer.KeyDeserializer.Invoke(reader); + serializer.KeyDeserializer.Invoke(reader, out var k); if (k != pair.Key) { Log.Warning("Broken index"); } - var v = serializer.ValueDeserializer.Invoke(reader); + serializer.ValueDeserializer.Invoke(reader, out var _); } } @@ -430,7 +430,7 @@ namespace PartitionFileStorageTest { try { - var key = serializer.KeyDeserializer.Invoke(reader); + serializer.KeyDeserializer.Invoke(reader, out var key); if (false == dict.ContainsKey(key)) { dict[key] = new HashSet(); @@ -439,7 +439,7 @@ namespace PartitionFileStorageTest { break; } - var input = serializer.InputDeserializer.Invoke(reader); + serializer.InputDeserializer.Invoke(reader, out var input); dict[key].Add(input); } catch (Exception ex) diff --git a/ZeroLevel/Services/PartitionStorage/Interfaces/IStoreSerializer.cs b/ZeroLevel/Services/PartitionStorage/Interfaces/IStoreSerializer.cs index 453092d..6132807 100644 --- a/ZeroLevel/Services/PartitionStorage/Interfaces/IStoreSerializer.cs +++ b/ZeroLevel/Services/PartitionStorage/Interfaces/IStoreSerializer.cs @@ -7,8 +7,8 @@ namespace ZeroLevel.Services.PartitionStorage.Interfaces { Action KeySerializer { get; } Action InputSerializer { get; } - Func KeyDeserializer { get; } - Func InputDeserializer { get; } - Func ValueDeserializer { get; } + TryDeserializeMethod KeyDeserializer { get; } + TryDeserializeMethod InputDeserializer { get; } + TryDeserializeMethod ValueDeserializer { get; } } } diff --git a/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs index a4b6732..7f5c2fc 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs @@ -60,6 +60,8 @@ namespace ZeroLevel.Services.PartitionStorage.Partition } public IEnumerable> Iterate() { + TKey key; + TInput input; var files = Directory.GetFiles(_catalog); if (files != null && files.Length > 0) { @@ -71,9 +73,9 @@ namespace ZeroLevel.Services.PartitionStorage.Partition { while (reader.EOS == false) { - var key = Serializer.KeyDeserializer.Invoke(reader); - var val = Serializer.InputDeserializer.Invoke(reader); - yield return new StorePartitionKeyValueSearchResult { Key = key, Value = val, Status = SearchResult.Success }; + if (Serializer.KeyDeserializer.Invoke(reader, out key) == false) break; + if (Serializer.InputDeserializer.Invoke(reader, out input) == false) break; + yield return new StorePartitionKeyValueSearchResult { Key = key, Value = input, Status = SearchResult.Success }; } } } @@ -119,12 +121,17 @@ namespace ZeroLevel.Services.PartitionStorage.Partition internal void CompressFile(string file) { + TKey key; + TInput input; var dict = new Dictionary>(); using (var reader = new MemoryStreamReader(new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.None, 4096 * 1024))) { while (reader.EOS == false) { - var key = Serializer.KeyDeserializer.Invoke(reader); + if (false == Serializer.KeyDeserializer.Invoke(reader, out key)) + { + throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read key."); + } if (false == dict.ContainsKey(key)) { dict[key] = new HashSet(); @@ -133,7 +140,10 @@ namespace ZeroLevel.Services.PartitionStorage.Partition { break; } - var input = Serializer.InputDeserializer.Invoke(reader); + if (false == Serializer.InputDeserializer.Invoke(reader, out input)) + { + throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault input value."); + } dict[key].Add(input); } } diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs index 566445a..3aa1fa3 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs @@ -31,15 +31,30 @@ namespace ZeroLevel.Services.PartitionStorage public StorePartitionKeyValueSearchResult Find(TKey key) { + TKey k; + TValue v; IViewAccessor memoryAccessor; - if (_options.Index.Enabled) + try { - var offset = Indexes.GetOffset(key); - memoryAccessor = offset.Length > 0 ? GetViewAccessor(key, offset.Offset, offset.Length) : GetViewAccessor(key, offset.Offset); + if (_options.Index.Enabled) + { + var offset = Indexes.GetOffset(key); + memoryAccessor = offset.Length > 0 ? GetViewAccessor(key, offset.Offset, offset.Length) : GetViewAccessor(key, offset.Offset); + } + else + { + memoryAccessor = GetViewAccessor(key, 0); + } } - else + catch (Exception ex) { - memoryAccessor = GetViewAccessor(key, 0); + Log.SystemError(ex, $"[StorePartitionAccessor.Find] Fault get IViewAccessor by key {(key == null ? string.Empty : key.ToString())}"); + return new StorePartitionKeyValueSearchResult + { + Key = key, + Status = SearchResult.FileLockedOrUnavaliable, + Value = default + }; } if (memoryAccessor != null) { @@ -47,8 +62,8 @@ namespace ZeroLevel.Services.PartitionStorage { while (reader.EOS == false) { - var k = Serializer.KeyDeserializer.Invoke(reader); - var v = Serializer.ValueDeserializer.Invoke(reader); + if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break; + if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break; var c = _options.KeyComparer(key, k); if (c == 0) return new StorePartitionKeyValueSearchResult { @@ -86,6 +101,8 @@ namespace ZeroLevel.Services.PartitionStorage } public IEnumerable> Iterate() { + TKey k; + TValue v; var files = Directory.GetFiles(_catalog); if (files != null && files.Length > 0) { @@ -98,8 +115,8 @@ namespace ZeroLevel.Services.PartitionStorage { while (reader.EOS == false) { - var k = Serializer.KeyDeserializer.Invoke(reader); - var v = Serializer.ValueDeserializer.Invoke(reader); + if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break; + if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break; yield return new StorePartitionKeyValueSearchResult { Key = k, Value = v, Status = SearchResult.Success }; } } @@ -109,6 +126,8 @@ namespace ZeroLevel.Services.PartitionStorage } public IEnumerable> IterateKeyBacket(TKey key) { + TKey k; + TValue v; var fileName = _options.GetFileName(key, _info); var filePath = Path.Combine(_catalog, fileName); if (File.Exists(filePath)) @@ -120,8 +139,8 @@ namespace ZeroLevel.Services.PartitionStorage { while (reader.EOS == false) { - var k = Serializer.KeyDeserializer.Invoke(reader); - var v = Serializer.ValueDeserializer.Invoke(reader); + if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break; + if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break; yield return new StorePartitionKeyValueSearchResult { Key = k, Value = v, Status = SearchResult.Success }; } } @@ -181,6 +200,8 @@ namespace ZeroLevel.Services.PartitionStorage private IEnumerable> Find(string fileName, TKey[] keys) { + TKey k; + TValue v; var filePath = Path.Combine(_catalog, fileName); if (_options.Index.Enabled) { @@ -204,8 +225,8 @@ namespace ZeroLevel.Services.PartitionStorage { while (reader.EOS == false) { - var k = Serializer.KeyDeserializer.Invoke(reader); - var v = Serializer.ValueDeserializer.Invoke(reader); + if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break; + if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break; var c = _options.KeyComparer(searchKey, k); if (c == 0) { @@ -237,8 +258,8 @@ namespace ZeroLevel.Services.PartitionStorage var keys_arr = keys.OrderBy(k => k).ToArray(); while (reader.EOS == false && index < keys_arr.Length) { - var k = Serializer.KeyDeserializer.Invoke(reader); - var v = Serializer.ValueDeserializer.Invoke(reader); + if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) break; + if (Serializer.ValueDeserializer.Invoke(reader, out v) == false) break; var c = _options.KeyComparer(keys_arr[index], k); if (c == 0) { @@ -269,6 +290,7 @@ namespace ZeroLevel.Services.PartitionStorage private void RemoveKeyGroup(string fileName, TKey[] keys, bool inverseRemove, bool autoReindex) { + TKey k; var filePath = Path.Combine(_catalog, fileName); // 1. Find ranges var ranges = new List(); @@ -295,8 +317,16 @@ namespace ZeroLevel.Services.PartitionStorage while (reader.EOS == false) { var startPosition = reader.Position; - var k = Serializer.KeyDeserializer.Invoke(reader); - Serializer.ValueDeserializer.Invoke(reader); + if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) + { + Log.Error($"[StorePartitionAccessor.RemoveKeyGroup] Fault remove keys from file '{fileName}'. Incorrect file structure. Fault read key."); + return; + } + if (Serializer.ValueDeserializer.Invoke(reader, out var _) == false) + { + Log.Error($"[StorePartitionAccessor.RemoveKeyGroup] Fault remove keys from file '{fileName}'. Incorrect file structure. Fault read value."); + return; + } var endPosition = reader.Position; var c = _options.KeyComparer(searchKey, k); if (c == 0) @@ -324,8 +354,16 @@ namespace ZeroLevel.Services.PartitionStorage while (reader.EOS == false && index < keys_arr.Length) { var startPosition = reader.Position; - var k = Serializer.KeyDeserializer.Invoke(reader); - Serializer.ValueDeserializer.Invoke(reader); + if (Serializer.KeyDeserializer.Invoke(reader, out k) == false) + { + Log.Error($"[StorePartitionAccessor.RemoveKeyGroup] Fault remove keys from file '{fileName}'. Incorrect file structure. Fault read key."); + return; + } + if (Serializer.ValueDeserializer.Invoke(reader, out var _) == false) + { + Log.Error($"[StorePartitionAccessor.RemoveKeyGroup] Fault remove keys from file '{fileName}'. Incorrect file structure. Fault read value."); + return; + } var endPosition = reader.Position; var c = _options.KeyComparer(keys_arr[index], k); if (c == 0) diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs index 8c03a38..ec11b9f 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs @@ -63,6 +63,8 @@ namespace ZeroLevel.Services.PartitionStorage } public IEnumerable> Iterate() { + TKey key; + TInput val; var files = Directory.GetFiles(_catalog); if (files != null && files.Length > 0) { @@ -75,8 +77,8 @@ namespace ZeroLevel.Services.PartitionStorage { while (reader.EOS == false) { - var key = Serializer.KeyDeserializer.Invoke(reader); - var val = Serializer.InputDeserializer.Invoke(reader); + if (Serializer.KeyDeserializer.Invoke(reader, out key) == false) break; + if (Serializer.InputDeserializer.Invoke(reader, out val) == false) break; yield return new StorePartitionKeyValueSearchResult { Key = key, Value = val, Status = SearchResult.Success }; } } @@ -135,6 +137,8 @@ namespace ZeroLevel.Services.PartitionStorage internal void CompressFile(string file) { + TKey key; + TInput input; var dict = new Dictionary>(); var accessor = PhisicalFileAccessorCachee.GetDataAccessor(file, 0); if (accessor != null) @@ -143,7 +147,10 @@ namespace ZeroLevel.Services.PartitionStorage { while (reader.EOS == false) { - var key = Serializer.KeyDeserializer.Invoke(reader); + if (Serializer.KeyDeserializer.Invoke(reader, out key) == false) + { + throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read key."); + } if (false == dict.ContainsKey(key)) { dict[key] = new HashSet(); @@ -152,7 +159,10 @@ namespace ZeroLevel.Services.PartitionStorage { break; } - var input = Serializer.InputDeserializer.Invoke(reader); + if (Serializer.InputDeserializer.Invoke(reader, out input) == false) + { + throw new Exception($"[StorePartitionBuilder.CompressFile] Fault compress data in file '{file}'. Incorrect file structure. Fault read input value."); + } dict[key].Add(input); } } diff --git a/ZeroLevel/Services/PartitionStorage/StoreStandartSerializer.cs b/ZeroLevel/Services/PartitionStorage/StoreStandartSerializer.cs index 973231f..5dcbc68 100644 --- a/ZeroLevel/Services/PartitionStorage/StoreStandartSerializer.cs +++ b/ZeroLevel/Services/PartitionStorage/StoreStandartSerializer.cs @@ -4,33 +4,36 @@ using ZeroLevel.Services.Serialization; namespace ZeroLevel.Services.PartitionStorage { + + // TODO INTERNAL public sealed class StoreStandartSerializer : IStoreSerializer { private readonly Action _keySerializer; private readonly Action _inputSerializer; - private readonly Func _keyDeserializer; - private readonly Func _inputDeserializer; - private readonly Func _valueDeserializer; + private readonly TryDeserializeMethod _keyDeserializer; + private readonly TryDeserializeMethod _inputDeserializer; + private readonly TryDeserializeMethod _valueDeserializer; public StoreStandartSerializer() { _keySerializer = MessageSerializer.GetSerializer(); _inputSerializer = MessageSerializer.GetSerializer(); - _keyDeserializer = MessageSerializer.GetDeserializer(); - _inputDeserializer = MessageSerializer.GetDeserializer(); - _valueDeserializer = MessageSerializer.GetDeserializer(); + + _keyDeserializer = MessageSerializer.GetSafetyDeserializer(); + _inputDeserializer = MessageSerializer.GetSafetyDeserializer(); + _valueDeserializer = MessageSerializer.GetSafetyDeserializer(); } public Action KeySerializer => _keySerializer; public Action InputSerializer => _inputSerializer; - public Func KeyDeserializer => _keyDeserializer; + public TryDeserializeMethod KeyDeserializer => _keyDeserializer; - public Func InputDeserializer => _inputDeserializer; + public TryDeserializeMethod InputDeserializer => _inputDeserializer; - public Func ValueDeserializer => _valueDeserializer; + public TryDeserializeMethod ValueDeserializer => _valueDeserializer; } } diff --git a/ZeroLevel/Services/Serialization/MemoryStreamReader.cs b/ZeroLevel/Services/Serialization/MemoryStreamReader.cs index 8735831..d943588 100644 --- a/ZeroLevel/Services/Serialization/MemoryStreamReader.cs +++ b/ZeroLevel/Services/Serialization/MemoryStreamReader.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Net; +using System.Runtime.Serialization; using System.Text; using ZeroLevel.Services.Extensions; using ZeroLevel.Services.Memory; @@ -203,6 +204,36 @@ namespace ZeroLevel.Services.Serialization return buffer; } + public bool TryReadBuffer(int count, out byte[] buffer) + { + if (CheckOutOfRange(count)) + { + buffer = null; + return false; + } + try + { + buffer = _accessor.ReadBuffer(count); + if (_reverseByteOrder && count > 1) + { + byte b; + for (int i = 0; i < (count >> 1); i++) + { + b = buffer[i]; + buffer[i] = buffer[count - i - 1]; + buffer[count - i - 1] = b; + } + } + } + catch (Exception ex) + { + Log.SystemError(ex, $"[MemoryStreamReader.TryReadBuffer] Fault read {count} bytes"); + buffer = null; + return false; + } + return true; + } + /// /// Reading the datetime /// @@ -1121,6 +1152,42 @@ namespace ZeroLevel.Services.Serialization return item; } + public bool TryReadByte(out byte b) + { + if (TryReadBuffer(1, out var buffer)) + { + b = buffer[0]; + return true; + } + b = default; + return false; + } + + public bool TryRead(out T item) where T : IBinarySerializable + { + if (TryReadByte(out var type)) + { + if (type == 0) + { + item = default(T); + return true; + } + try + { + var o = (IBinarySerializable)FormatterServices.GetUninitializedObject(typeof(T)); + o.Deserialize(this); + item = (T)o; + return true; + } + catch (Exception ex) + { + Log.SystemError(ex, "[MemoryStreamReader.TryRead]"); + } + } + item = default; + return false; + } + public T Read(object arg) where T : IBinarySerializable { byte type = ReadByte(); diff --git a/ZeroLevel/Services/Serialization/MessageSerializer.cs b/ZeroLevel/Services/Serialization/MessageSerializer.cs index e6b662d..4cd16d0 100644 --- a/ZeroLevel/Services/Serialization/MessageSerializer.cs +++ b/ZeroLevel/Services/Serialization/MessageSerializer.cs @@ -1,11 +1,11 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Runtime.Serialization; namespace ZeroLevel.Services.Serialization { + public delegate bool TryDeserializeMethod(MemoryStreamReader reader, out T output); public static class MessageSerializer { public static byte[] Serialize(T obj) @@ -54,6 +54,47 @@ namespace ZeroLevel.Services.Serialization return (r) => PrimitiveTypeSerializer.Deserialize(r); } + static bool TryObjectDeserialize(MemoryStreamReader reader, out T output) + { + try + { + var o = (IBinarySerializable)FormatterServices.GetUninitializedObject(typeof(T)); + o.Deserialize(reader); + output = (T)o; + return true; + } + catch (Exception ex) + { + Log.SystemError(ex, $"[MessageSerializer.TryObjectDeserialize] Fault deserialize type {typeof(T).Name}"); + output = default; + } + return false; + } + + static bool TryPrimitiveTypeDeserialize(MemoryStreamReader reader, out T output) + { + try + { + output = PrimitiveTypeSerializer.Deserialize(reader); + return true; + } + catch (Exception ex) + { + Log.SystemError(ex, $"[MessageSerializer.TryPrimitiveTypeDeserialize] Fault deserialize type {typeof(T).Name}"); + output = default; + } + return false; + } + + public static TryDeserializeMethod GetSafetyDeserializer() + { + if (typeof(IBinarySerializable).IsAssignableFrom(typeof(T))) + { + return TryObjectDeserialize; + } + return TryPrimitiveTypeDeserialize; + } + public static byte[] SerializeCompatible(object obj) { if (null == obj) diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index 712cfbe..3a731ce 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -6,16 +6,16 @@ ogoun ogoun - 3.3.8.8 - MMF for partition storage + 3.3.8.9 + Partition storage. Suppress exception when find invoke https://github.com/ogoun/Zero/wiki Copyright Ogoun 2023 https://github.com/ogoun/Zero git - 3.3.8.8 - 3.3.8.8 + 3.3.8.9 + 3.3.8.9 AnyCPU;x64;x86 zero.png full