diff --git a/.gitignore b/.gitignore index 73838a3..27697d7 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ bower_components *.nupkg *.p7s *.sln.ide +*.onnx [Bb]in [Dd]ebug*/ [Oo]bj/ diff --git a/AutoLoader/AppContainer/AppContainer.csproj b/AutoLoader/AppContainer/AppContainer.csproj new file mode 100644 index 0000000..132c02c --- /dev/null +++ b/AutoLoader/AppContainer/AppContainer.csproj @@ -0,0 +1,9 @@ + + + + net6.0 + enable + enable + + + diff --git a/AutoLoader/AppContainer/Class1.cs b/AutoLoader/AppContainer/Class1.cs new file mode 100644 index 0000000..74f85db --- /dev/null +++ b/AutoLoader/AppContainer/Class1.cs @@ -0,0 +1,50 @@ +namespace AppContainer +{ + + public class Endpoint + { + public string VersionEndpoint { get; set; } + public string AppEndpoint { get; set; } + public string PingEndpoint { get; set; } + public string Token { get; set; } + } + public class HostSettings + { + public IEnumerable Endpoints { get; set; } + public string EntryClass { get; set; } + public string EntryMethod { get; set; } + } + + public class Host + { + private readonly HostSettings _settings; + public Host(HostSettings settings) + { + if (settings == null) throw new ArgumentNullException(nameof(settings)); + _settings = settings; + } + + public async Task HasUpdate() + { + if (_settings.Endpoints != null) + { + foreach (var ep in _settings.Endpoints) + { + try + { + using (var client = new HttpClient()) + { + client.DefaultRequestHeaders.Add("token", ep.Token); + var response = await client.GetAsync(ep.PingEndpoint); + response.EnsureSuccessStatusCode(); + + } + } + catch (Exception ex) + { + } + } + } + } + } +} \ No newline at end of file diff --git a/ConnectionTest/Client/Client.csproj b/ConnectionTest/Client/Client.csproj index 9fbf72d..fc81762 100644 --- a/ConnectionTest/Client/Client.csproj +++ b/ConnectionTest/Client/Client.csproj @@ -1,9 +1,10 @@ - + Exe net6.0 - AnyCPU;x64;x86 + enable + enable diff --git a/ConnectionTest/Client/Program.cs b/ConnectionTest/Client/Program.cs index 13e8ba7..d8fc73f 100644 --- a/ConnectionTest/Client/Program.cs +++ b/ConnectionTest/Client/Program.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; +using System.Net; using ZeroLevel; using ZeroLevel.Network; using ZeroLevel.Services.HashFunctions; @@ -60,6 +57,9 @@ namespace Client { private readonly static XXHashUnsafe _hash = new XXHashUnsafe(667); + private static long _successSend = 0; + private static long _faultSend = 0; + static void Main(string[] args) { Log.AddConsoleLogger(); @@ -68,6 +68,12 @@ namespace Client var port = ReadPort(); ex.RoutesStorage.Set("server", new IPEndPoint(address, port)); + Sheduller.RemindEvery(TimeSpan.FromMilliseconds(300), () => + { + Console.SetCursorPosition(0, 0); + Console.WriteLine($"Success/Fault network: {_successSend} / {_faultSend}"); + }); + uint index = 0; while (true) { @@ -99,7 +105,7 @@ namespace Client var info = new Info { Checksum = full_checksum, Id = id, Length = length }; if (client.Request("start", info, res => { - Log.Info($"Success start sending packet '{id}'"); + Interlocked.Increment(ref _successSend); })) { uint size = 1; @@ -111,11 +117,15 @@ namespace Client { if (!res) { - Log.Info($"Fault server incoming packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes."); + Interlocked.Increment(ref _faultSend); + } + else + { + Interlocked.Increment(ref _successSend); } })) { - Log.Warning($"Can't start send packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes. No connection"); + Interlocked.Increment(ref _faultSend); } offset += size; size += 1; @@ -124,7 +134,7 @@ namespace Client } else { - Log.Warning($"Can't start send packet '{id}'. No connection"); + Interlocked.Increment(ref _faultSend); } } @@ -135,7 +145,7 @@ namespace Client var info = new Info { Checksum = full_checksum, Id = id, Length = length }; if (exchange.Request("server", "start", info)) { - Log.Info($"Success start sending packet '{id}'"); + Interlocked.Increment(ref _successSend); uint size = 4096; uint offset = 0; while (offset < payload.Length) @@ -143,7 +153,11 @@ namespace Client var fragment = GetFragment(id, payload, offset, size); if (!exchange.Request("server", "part", fragment)) { - Log.Info($"Fault server incoming packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes."); + Interlocked.Increment(ref _faultSend); + } + else + { + Interlocked.Increment(ref _successSend); } offset += size; } @@ -151,7 +165,7 @@ namespace Client } else { - Log.Warning($"Can't start send packet '{id}'. No connection"); + Interlocked.Increment(ref _faultSend); } } diff --git a/ConnectionTest/Client/Properties/PublishProfiles/FolderProfile.pubxml b/ConnectionTest/Client/Properties/PublishProfiles/FolderProfile.pubxml deleted file mode 100644 index ccb19b1..0000000 --- a/ConnectionTest/Client/Properties/PublishProfiles/FolderProfile.pubxml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - FileSystem - Release - Any CPU - netcoreapp3.1 - bin\Release\netcoreapp3.1\publish\ - - \ No newline at end of file diff --git a/ConnectionTest/Server/Program.cs b/ConnectionTest/Server/Program.cs index 40701f1..75657cc 100644 --- a/ConnectionTest/Server/Program.cs +++ b/ConnectionTest/Server/Program.cs @@ -6,6 +6,7 @@ using ZeroLevel.Services.Serialization; namespace Server { + //netsh advfirewall firewall add rule name="ClientTest" dir=in action=allow protocol=TCP localport=5016 public class Info : IBinarySerializable { diff --git a/TestApp/HybrydLock.cs b/TestApp/HybrydLock.cs deleted file mode 100644 index abc7bc9..0000000 --- a/TestApp/HybrydLock.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Threading; - -namespace TestApp -{ - public class HybrydLock - { - private AutoResetEvent _lock = new AutoResetEvent(false); - private volatile int _counter = 0; - - public void Enter() - { - if (Interlocked.Increment(ref _counter) == 1) - { - return; - } - _lock.WaitOne(); - } - - public void Leave() - { - if (Interlocked.Decrement(ref _counter) == 0) - { - return; - } - _lock.Set(); - } - } -} diff --git a/TestApp/MyService.cs b/TestApp/MyService.cs deleted file mode 100644 index 213cf4e..0000000 --- a/TestApp/MyService.cs +++ /dev/null @@ -1,72 +0,0 @@ -using System; -using System.Net; -using System.Threading; -using ZeroLevel; -using ZeroLevel.Network; -using ZeroLevel.Services.Applications; - -namespace TestApp -{ - public class MyService - : BaseZeroService - { - public MyService() - : base() - { - } - - protected override void StartAction() - { - Log.Info("Started"); - ReadServiceInfo(); - var host = UseHost(8800); - AutoregisterInboxes(host); - host.OnConnect += Host_OnConnect; - host.OnDisconnect += Host_OnDisconnect; - - int counter = 0; - Sheduller.RemindEvery(TimeSpan.FromSeconds(1), () => - { - Log.Info($"RPS: {counter}"); - Interlocked.Exchange(ref counter, 0); - }); - - Exchange.RoutesStorage.Set("test.app", new IPEndPoint(IPAddress.Loopback, 8800)); - - while (true) - { - try - { - //var s = Exchange.Request("test.app", "counter"); - Interlocked.Add(ref counter, Exchange.Request("test.app", "counter")); - } - catch(Exception ex) - { - Log.Error(ex, "Request fault"); - Thread.Sleep(300); - } - } - } - - private void Host_OnDisconnect(ISocketClient obj) - { - Log.Info($"Client '{obj.Endpoint.Address}:{obj.Endpoint.Port}' disconnected"); - } - - private void Host_OnConnect(IClient obj) - { - Log.Info($"Client '{obj.Socket.Endpoint.Address}:{obj.Socket.Endpoint.Port}' connected"); - } - - [ExchangeReplierWithoutArg("counter")] - public int GetCounter(ISocketClient client) - { - return 1; - } - - protected override void StopAction() - { - Log.Info("Stopped"); - } - } -} \ No newline at end of file diff --git a/TestApp/PersonDetector.cs b/TestApp/PersonDetector.cs new file mode 100644 index 0000000..a3415c8 --- /dev/null +++ b/TestApp/PersonDetector.cs @@ -0,0 +1,44 @@ +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; +using System.Collections.Generic; +using System.Linq; +using ZeroLevel.NN.Architectures.YoloV5; +using ZeroLevel.NN.Models; +using ZeroLevel.NN.Services; + +namespace TestApp +{ + public class PersonDetector + { + private const string MODEL_PATH = @"nnmodels/Yolo5S/yolov5s327e.onnx"; + private readonly Yolov5Detector _detector; + private float _threshold = 0.17f; + public PersonDetector() + { + _detector = new Yolov5Detector(MODEL_PATH, gpu: false); + } + + public IEnumerable Detect(string imagePath) + { + using (Image image = Image.Load(imagePath)) + { + var t_predictions = _detector.PredictMultiply(image, true, _threshold); + t_predictions.Apply(p => + { + p.Cx /= image.Width; + p.Cy /= image.Height; + }); + if (t_predictions != null) + { + t_predictions.RemoveAll(p => (p.W * image.Width) < 10.0f || (p.H * image.Height) < 10.0f); + } + if (t_predictions.Count > 0) + { + NMS.Apply(t_predictions); + return t_predictions; + } + } + return Enumerable.Empty(); + } + } +} diff --git a/TestApp/Program.cs b/TestApp/Program.cs index 593761c..b4b4a9b 100644 --- a/TestApp/Program.cs +++ b/TestApp/Program.cs @@ -1,84 +1,11 @@ -using Newtonsoft.Json; -using System; -using System.Net; -using System.Threading; -using ZeroLevel; -using ZeroLevel.Logging; -using ZeroLevel.Network; -using ZeroLevel.Services.Serialization; -using ZeroLevel.Services.Trees; - -namespace TestApp +namespace TestApp { - public class TestQuery - { - public string Name { get; set; } - public int Age { get; set; } - public string[] Roles { get; set; } - } - internal static class Program { - private static string Serialize(object instance) - { - return JsonConvert.SerializeObject(instance); - } - private static void Main(string[] args) { - Configuration.Save(Configuration.ReadFromApplicationConfig()); - Bootstrap.Startup(args, - () => Configuration.ReadSetFromIniFile("config.ini")) - .EnableConsoleLog(LogLevel.System | LogLevel.FullDebug) - //.UseDiscovery() - .Run() - .WaitWhileStatus(ZeroServiceStatus.Running) - .Stop(); - Bootstrap.Shutdown(); - } - - static void SimpleCSTest() - { - var server_router = new Router(); - server_router.RegisterInbox("test", (c, line) => - { - Console.WriteLine(line); - }); - - server_router.RegisterInbox("req", (c, line) => - { - Console.WriteLine($"Request: {line}"); - return line.ToUpperInvariant(); - }); - - var server = new SocketServer(new System.Net.IPEndPoint(IPAddress.Any, 666), server_router); - - - var client_router = new Router(); - var client = new SocketClient(new IPEndPoint(IPAddress.Loopback, 666), client_router); - - var frm = FrameFactory.Create("req", MessageSerializer.SerializeCompatible("Hello world")); - - while (Console.KeyAvailable == false) - { - client.Request(frm, data => - { - var line = MessageSerializer.DeserializeCompatible(data); - Console.WriteLine($"Response: {line}"); - }); - Thread.Sleep(2000); - } - } - - public static double[] Generate(int vector_size) - { - var rnd = new Random((int)Environment.TickCount); - var vector = new double[vector_size]; - for (int i = 0; i < vector_size; i++) - { - vector[i] = 50.0d - rnd.NextDouble() * 100.0d; - } - return vector; + var detector = new PersonDetector(); + var predictions = detector.Detect(@"E:\Desktop\test\1.JPG"); } } } \ No newline at end of file diff --git a/TestApp/TestApp.csproj b/TestApp/TestApp.csproj index 844a5fb..480e7bd 100644 --- a/TestApp/TestApp.csproj +++ b/TestApp/TestApp.csproj @@ -11,6 +11,7 @@ + @@ -18,6 +19,9 @@ Always + + PreserveNewest + diff --git a/ZeroLevel.NN/Architectures/AgeDetectors/GenderAgeEstimator.cs b/ZeroLevel.NN/Architectures/AgeDetectors/GenderAgeEstimator.cs new file mode 100644 index 0000000..8315304 --- /dev/null +++ b/ZeroLevel.NN/Architectures/AgeDetectors/GenderAgeEstimator.cs @@ -0,0 +1,37 @@ +using Microsoft.ML.OnnxRuntime.Tensors; +using SixLabors.ImageSharp; +using ZeroLevel.NN.Models; + +namespace ZeroLevel.NN +{ + public class GenderAgeEstimator + : SSDNN + { + private const int INPUT_WIDTH = 64; + private const int INPUT_HEIGHT = 64; + + public GenderAgeEstimator(string modelPath, bool gpu = false) : base(modelPath, gpu) + { + } + + public (Gender, int) Predict(Image image) + { + var input = MakeInput(image, + new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst) + .ApplyNormilization() + .ApplyAxeInversion()); + return Predict(input); + } + + public (Gender, int) Predict(Tensor input) + { + float[] variances = null; + Extract(new Dictionary> { { "input", input } }, d => + { + variances = d.First().Value.ToArray(); + }); + var gender = Argmax(variances[0..2]) == 0 ? Gender.Male : Gender.Feemale; + return (gender, (int)variances[2]); + } + } +} diff --git a/ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeDetector.cs b/ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeEstimator.cs similarity index 88% rename from ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeDetector.cs rename to ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeEstimator.cs index 04f0d38..53f3a6d 100644 --- a/ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeDetector.cs +++ b/ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeEstimator.cs @@ -19,7 +19,7 @@ namespace ZeroLevel.NN /// /// Input tensor is 1 x 3 x height x width with mean values 104, 117, 123. Input image have to be previously resized to 224 x 224 pixels and converted to BGR format. /// - public class GoogleAgeDetector + public class GoogleAgeEstimator : SSDNN { private const int INPUT_WIDTH = 224; @@ -28,7 +28,7 @@ namespace ZeroLevel.NN private Age[] _ageList = new[] { Age.From0To2, Age.From4To6, Age.From8To12, Age.From15To20, Age.From25To32, Age.From38To43, Age.From48To53, Age.From60To100 }; - public GoogleAgeDetector(string modelPath, bool gpu = false) : base(modelPath, gpu) + public GoogleAgeEstimator(string modelPath, bool gpu = false) : base(modelPath, gpu) { } @@ -48,7 +48,7 @@ namespace ZeroLevel.NN { variances = d.First().Value.ToArray(); }); - var (number, index) = variances.Select((n, i) => (n, i)).Max(); + var index = Argmax(variances); return _ageList[index]; } } diff --git a/ZeroLevel.NN/Architectures/FaceSeacrhService.cs b/ZeroLevel.NN/Architectures/FaceSeacrhService.cs index 68f79d9..68543a7 100644 --- a/ZeroLevel.NN/Architectures/FaceSeacrhService.cs +++ b/ZeroLevel.NN/Architectures/FaceSeacrhService.cs @@ -1,6 +1,7 @@ using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; +using ZeroLevel; using ZeroLevel.NN; using ZeroLevel.NN.Models; @@ -127,25 +128,36 @@ namespace Zero.NN.Services var around_x2 = centerFaceX + radius; var around_y1 = centerFaceY - radius; var around_y2 = centerFaceY + radius; - using (var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2)) + try { - var matrix = Face.GetTransformMatrix(face); - var builder = new AffineTransformBuilder(); - builder.AppendMatrix(matrix); - faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic)); - vector = _encoder.Predict(faceImage); - /*var aligned_faces = detector.Predict(faceImage); - if (aligned_faces != null && aligned_faces.Count == 1) + using (var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2)) { - using (var ci = SpecialCrop(faceImage, aligned_faces[0])) + var matrix = Face.GetTransformMatrix(face); + var builder = new AffineTransformBuilder(); + builder.AppendMatrix(matrix); + faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic)); + vector = _encoder.Predict(faceImage); + /*var aligned_faces = detector.Predict(faceImage); + if (aligned_faces != null && aligned_faces.Count == 1) { - vector = encoder.Predict(faceImage); + using (var ci = SpecialCrop(faceImage, aligned_faces[0])) + { + vector = encoder.Predict(faceImage); + } } + else + { + vector = encoder.Predict(faceImage); + }*/ } - else + } + catch (Exception ex) + { + Log.SystemError(ex, "[FaceSeacrhService.GetEmbeddings]"); + using (var faceImage = ImagePreprocessor.Crop(image, face.X1, face.Y1, face.X2, face.Y2)) { - vector = encoder.Predict(faceImage); - }*/ + vector = _encoder.Predict(faceImage); + } } } else @@ -162,5 +174,53 @@ namespace Zero.NN.Services }; } } + + + public IEnumerable<(FaceEmbedding, Image)> GetEmbeddingsAndCrop(Image image) + { + int width = image.Width; + int heigth = image.Height; + var faces = _detector.Predict(image); + foreach (var face in faces) + { + Face.FixInScreen(face, width, heigth); + float[] vector; + if (_useFaceAlign) + { + int x = (int)face.X1; + int y = (int)face.Y1; + int w = (int)(face.X2 - face.X1); + int h = (int)(face.Y2 - face.Y1); + var radius = (float)Math.Sqrt(w * w + h * h) / 2f; + var centerFaceX = (face.X2 + face.X1) / 2.0f; + var centerFaceY = (face.Y2 + face.Y1) / 2.0f; + var around_x1 = centerFaceX - radius; + var around_x2 = centerFaceX + radius; + var around_y1 = centerFaceY - radius; + var around_y2 = centerFaceY + radius; + var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2); + var matrix = Face.GetTransformMatrix(face); + var builder = new AffineTransformBuilder(); + builder.AppendMatrix(matrix); + faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic)); + vector = _encoder.Predict(faceImage); + yield return (new FaceEmbedding + { + Face = face, + Vector = vector + }, faceImage); + } + else + { + var faceImage = ImagePreprocessor.Crop(image, face.X1, face.Y1, face.X2, face.Y2); + vector = _encoder.Predict(faceImage); + yield return (new FaceEmbedding + { + Face = face, + Vector = vector + }, faceImage); + } + } + } } } diff --git a/ZeroLevel.NN/Architectures/YoloV5/Yolov5Detector.cs b/ZeroLevel.NN/Architectures/YoloV5/Yolov5Detector.cs new file mode 100644 index 0000000..855dc43 --- /dev/null +++ b/ZeroLevel.NN/Architectures/YoloV5/Yolov5Detector.cs @@ -0,0 +1,144 @@ +using Microsoft.ML.OnnxRuntime.Tensors; +using SixLabors.ImageSharp; +using ZeroLevel.NN.Models; + +namespace ZeroLevel.NN.Architectures.YoloV5 +{ + public class Yolov5Detector + : SSDNN + { + private int INPUT_WIDTH = 640; + private int INPUT_HEIGHT = 640; + private int CROP_WIDTH = 1440; + private int CROP_HEIGHT = 1440; + + public Yolov5Detector(string modelPath, int inputWidth = 640, int inputHeight = 640, bool gpu = false) + : base(modelPath, gpu) + { + INPUT_HEIGHT = inputHeight; + INPUT_WIDTH = inputWidth; + } + + public List Predict(Image image, float threshold) + { + var input = MakeInput(image, + new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst) + .ApplyNormilization() + .ApplyAxeInversion()); + return Predict(input, threshold); + } + + public List PredictMultiply(Image image, bool withFullResizedImage, float threshold) + { + var input = MakeInputBatch(image, + new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst) + .ApplyNormilization() + .ApplyAxeInversion() + .UseCrop(CROP_WIDTH, CROP_HEIGHT, withFullResizedImage, true)); + return PredictMultiply(input, threshold); + } + + public List Predict(Tensor input, float threshold) + { + var result = new List(); + Extract(new Dictionary> { { "images", input } }, d => + { + var output = d["output"]; + /* + var output350 = d["350"]; + var output498 = d["498"]; + var output646 = d["646"]; + */ + if (output != null && output != null) + { + var relative_koef_x = 1.0f / INPUT_WIDTH; + var relative_koef_y = 1.0f / INPUT_HEIGHT; + for (int box = 0; box < output.Dimensions[1]; box++) + { + var conf = output[0, box, 4]; // уверенность в наличии любого объекта + if (conf > threshold) + { + var class_confidense = output[0, box, 5]; // уверенность в наличии объекта класса person + if (class_confidense > threshold) + { + // Перевод относительно входа модели в относительные координаты + var cx = output[0, box, 0] * relative_koef_x; + var cy = output[0, box, 1] * relative_koef_y; + var h = output[0, box, 2] * relative_koef_y; + var w = output[0, box, 3] * relative_koef_x; + result.Add(new YoloPrediction + { + Cx = cx, + Cy = cy, + W = w, + H = h, + Class = 0, + Label = "0", + Score = conf + }); + } + } + } + } + }); + return result; + } + + public List PredictMultiply(ImagePredictionInput[] inputs, float threshold) + { + var result = new List(); + var relative_koef_x = 1.0f / (float)INPUT_WIDTH; + var relative_koef_y = 1.0f / (float)INPUT_HEIGHT; + foreach (var input in inputs) + { + Extract(new Dictionary> { { "images", input.Tensor } }, d => + { + var output = d["output"]; + /* + var output350 = d["350"]; + var output498 = d["498"]; + var output646 = d["646"]; + */ + if (output != null && output != null) + { + for (int index = 0; index < input.Count; index++) + { + var real_koef_x = (float)input.Offsets[index].Width / (float)INPUT_WIDTH; + var real_koef_y = (float)input.Offsets[index].Height / (float)INPUT_HEIGHT; + for (int box = 0; box < output.Dimensions[1]; box++) + { + var conf = output[index, box, 4]; // уверенность в наличии любого объекта + if (conf > threshold) + { + var class_confidense = output[index, box, 5]; // уверенность в наличии объекта класса person + if (class_confidense > threshold) + { + // Перевод относительно входа модели в относительные координаты + var cx = output[index, box, 0] * real_koef_x; + var cy = output[index, box, 1] * real_koef_y; + var h = output[index, box, 2] * relative_koef_y; + var w = output[index, box, 3] * relative_koef_x; + // Перевод в координаты отнисительно текущего смещения + cx += input.Offsets[index].X; + cy += input.Offsets[index].Y; + result.Add(new YoloPrediction + { + Cx = cx, + Cy = cy, + W = w, + H = h, + Class = 0, + Label = "0", + Score = conf + }); + } + } + } + } + } + }); + } + return result; + } + } +} diff --git a/ZeroLevel.NN/Models/Gender.cs b/ZeroLevel.NN/Models/Gender.cs new file mode 100644 index 0000000..8fc5795 --- /dev/null +++ b/ZeroLevel.NN/Models/Gender.cs @@ -0,0 +1,9 @@ +namespace ZeroLevel.NN.Models +{ + public enum Gender + { + Unknown = 0, + Male = 1, + Feemale = 2 + } +} diff --git a/ZeroLevel.NN/Models/PredictionInput.cs b/ZeroLevel.NN/Models/ImagePredictionInput.cs similarity index 83% rename from ZeroLevel.NN/Models/PredictionInput.cs rename to ZeroLevel.NN/Models/ImagePredictionInput.cs index 438f42f..3934037 100644 --- a/ZeroLevel.NN/Models/PredictionInput.cs +++ b/ZeroLevel.NN/Models/ImagePredictionInput.cs @@ -2,7 +2,7 @@ namespace ZeroLevel.NN.Models { - public class PredictionInput + public class ImagePredictionInput { public Tensor Tensor; public OffsetBox[] Offsets; diff --git a/ZeroLevel.NN/Models/ImagePreprocessorOptions.cs b/ZeroLevel.NN/Models/ImagePreprocessorOptions.cs index 2566676..91e783f 100644 --- a/ZeroLevel.NN/Models/ImagePreprocessorOptions.cs +++ b/ZeroLevel.NN/Models/ImagePreprocessorOptions.cs @@ -2,7 +2,7 @@ { public class ImagePreprocessorOptions { - private const float PIXEL_NORMALIZATION_SCALE = 1f / 255f; + private const float PIXEL_NORMALIZATION_SCALE = 1.0f / 255.0f; public ImagePreprocessorOptions(int inputWidth, int inputHeight, PredictorChannelType channelType) { this.InputWidth = inputWidth; diff --git a/ZeroLevel.NN/Models/YoloPrediction.cs b/ZeroLevel.NN/Models/YoloPrediction.cs new file mode 100644 index 0000000..2ae7f87 --- /dev/null +++ b/ZeroLevel.NN/Models/YoloPrediction.cs @@ -0,0 +1,66 @@ +using ZeroLevel.Services.Serialization; + +namespace ZeroLevel.NN.Models +{ + public class YoloPrediction + : IBinarySerializable + { + public int Class { get; set; } + public float Cx { get; set; } + public float Cy { get; set; } + public float W { get; set; } + public float H { get; set; } + public float Score { get; set; } + public string Label { get; set; } + + public float X { get { return Cx - W / 2.0f; } } + public float Y { get { return Cy - W / 2.0f; } } + + public float Area { get { return W * H; } } + + public string Description + { + get + { + return $"{Label} ({(int)(Score * 100)} %)"; + } + } + + public float this[int index] + { + get + { + switch (index) + { + case 0: return Cx; + case 1: return Cy; + case 2: return Cx + W; + case 3: return Cy + H; + } + return 0; + } + } + + public void Deserialize(IBinaryReader reader) + { + this.Cx = reader.ReadFloat(); + this.Cy = reader.ReadFloat(); + this.W = reader.ReadFloat(); + this.H = reader.ReadFloat(); + this.Class = reader.ReadInt32(); + this.Score = reader.ReadFloat(); + this.Label = reader.ReadString(); + } + + public void Serialize(IBinaryWriter writer) + { + writer.WriteFloat(this.Cx); + writer.WriteFloat(this.Cy); + writer.WriteFloat(this.W); + writer.WriteFloat(this.H); + writer.WriteInt32(this.Class); + writer.WriteFloat(this.Score); + writer.WriteString(this.Label); + } + } +} diff --git a/ZeroLevel.NN/Services/ImagePreprocessor.cs b/ZeroLevel.NN/Services/ImagePreprocessor.cs index b5e824e..4a07855 100644 --- a/ZeroLevel.NN/Services/ImagePreprocessor.cs +++ b/ZeroLevel.NN/Services/ImagePreprocessor.cs @@ -8,6 +8,8 @@ namespace ZeroLevel.NN { public static class ImagePreprocessor { + private static Action, float, int, int, int, int> _precompiledChannelFirstAction = new Action, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, c, i, j] = v; }); + private static Action, float, int, int, int, int> _precompiledChannelLastAction = new Action, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, i, j, c] = v; }); private static Func PixelToTensorMethod(ImagePreprocessorOptions options) { if (options.Normalize) @@ -42,26 +44,51 @@ namespace ZeroLevel.NN return new Func((b, _) => (float)b); } + //private static int CalculateFragmentsCount(Image image, ImagePreprocessorOptions options) + //{ + // int count = (options.Crop.SaveOriginal ? 1 : 0); + // var Sw = image.Width; // ширина оригинала + // var Sh = image.Height; // высота оригинала + + // var CRw = options.Crop.Width; // ширина кропа + // var CRh = options.Crop.Height; // высота кропа + + // var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw; // сдвиг по OX к следующему кропу + // var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh; // сдвиг по OY к следующему кропу + + // for (int x = 0; x < Sw; x += Dx) + // { + // for (int y = 0; y < Sh; y += Dy) + // { + // count++; + // } + // } + // return count; + //} private static int CalculateFragmentsCount(Image image, ImagePreprocessorOptions options) { - int count = 0; - var xs = options.Crop.Overlap ? (int)(options.Crop.Width * options.Crop.OverlapKoefWidth) : options.Crop.Width; - var ys = options.Crop.Overlap ? (int)(options.Crop.Height * options.Crop.OverlapKoefHeight) : options.Crop.Height; - for (var x = 0; x < image.Width - xs; x += xs) + int count = (options.Crop.SaveOriginal ? 1 : 0); + var Sw = image.Width; // ширина оригинала + var Sh = image.Height; // высота оригинала + + var CRw = options.InputWidth; // ширина кропа (равна ширине входа, т.к. изображение отресайзено подобающим образом) + var CRh = options.InputHeight; // высота кропа (равна высоте входа, т.к. изображение отресайзено подобающим образом) + + var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw; // сдвиг по OX к следующему кропу + var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh; // сдвиг по OY к следующему кропу + + for (int x = 0; x < Sw; x += Dx) { - for (var y = 0; y < image.Height - ys; y += ys) + for (int y = 0; y < Sh; y += Dy) { count++; } } return count; } - private static void FillTensor(Tensor tensor, Image image, int index, ImagePreprocessorOptions options, Func pixToTensor) { - var append = options.ChannelType == PredictorChannelType.ChannelFirst - ? new Action, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, c, i, j] = v; }) - : new Action, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, i, j, c] = v; }); + var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction; ((Image)image).ProcessPixelRows(pixels => { @@ -112,6 +139,59 @@ namespace ZeroLevel.NN }); } + private static void FillTensor(Tensor tensor, Image image, int startX, int startY, int w, int h, int index, ImagePreprocessorOptions options, Func pixToTensor) + { + var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction; + + ((Image)image).ProcessPixelRows(pixels => + { + if (options.InvertXY) + { + for (int y = startY; y < h; y++) + { + Span pixelSpan = pixels.GetRowSpan(y); + for (int x = startX; x < w; x++) + { + if (options.BGR) + { + append(tensor, pixToTensor(pixelSpan[x].B, 0), index, 0, y, x); + append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, y, x); + append(tensor, pixToTensor(pixelSpan[x].R, 2), index, 2, y, x); + } + else + { + append(tensor, pixToTensor(pixelSpan[x].R, 0), index, 0, y, x); + append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, y, x); + append(tensor, pixToTensor(pixelSpan[x].B, 2), index, 2, y, x); + } + } + } + } + else + { + for (int y = startY; y < h; y++) + { + Span pixelSpan = pixels.GetRowSpan(y); + for (int x = startX; x < w; x++) + { + if (options.BGR) + { + append(tensor, pixToTensor(pixelSpan[x].B, 0), index, 0, x, y); + append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, x, y); + append(tensor, pixToTensor(pixelSpan[x].R, 2), index, 2, x, y); + } + else + { + append(tensor, pixToTensor(pixelSpan[x].R, 0), index, 0, x, y); + append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, x, y); + append(tensor, pixToTensor(pixelSpan[x].B, 2), index, 2, x, y); + } + } + } + } + }); + } + private static Tensor InitInputTensor(ImagePreprocessorOptions options, int batchSize = 1) { switch (options.ChannelType) @@ -127,101 +207,139 @@ namespace ZeroLevel.NN } } - public static PredictionInput[] ToTensors(this Image image, ImagePreprocessorOptions options) + + public static ImagePredictionInput[] ToTensors(this Image image, ImagePreprocessorOptions options) { - PredictionInput[] result = null; + ImagePredictionInput[] result = null; var pixToTensor = PixelToTensorMethod(options); options.Channels = image.PixelType.BitsPerPixel >> 3; - if (options.Crop.Enabled) { - var fragments = CalculateFragmentsCount(image, options); - int count = CalculateFragmentsCount(image, options) + (options.Crop.SaveOriginal ? 1 : 0); - int offset = count % options.MaxBatchSize; - int count_tensors = count / options.MaxBatchSize + (offset == 0 ? 0 : 1); - var tensors = new PredictionInput[count_tensors]; - for (int i = 0; i < count_tensors; i++) - { - if (i < count_tensors - 1) - { - tensors[i] = new PredictionInput - { - Tensor = InitInputTensor(options, options.MaxBatchSize), - Offsets = new OffsetBox[options.MaxBatchSize], - Count = options.MaxBatchSize - }; - } - else - { - tensors[i] = new PredictionInput - { - Tensor = InitInputTensor(options, offset == 0 ? options.MaxBatchSize : offset), - Offsets = new OffsetBox[offset == 0 ? options.MaxBatchSize : offset], - Count = offset == 0 ? options.MaxBatchSize : offset - }; - } - } + // Размеры оригинального изображения + var Sw = image.Width; + var Sh = image.Height; - int tensor_index = 0; - int tensor_part_index = 0; - var xs = options.Crop.Overlap ? (int)(options.Crop.Width * options.Crop.OverlapKoefWidth) : options.Crop.Width; - var ys = options.Crop.Overlap ? (int)(options.Crop.Height * options.Crop.OverlapKoefHeight) : options.Crop.Height; + // Создание ресайза для целочисленного прохода кропами шириной CRw и высотой CRh + var resizedForCropWidthKoef = options.InputWidth / (double)options.Crop.Width; + var resizedForCropHeightKoef = options.InputHeight / (double)options.Crop.Height; - if (options.Crop.SaveOriginal) - { - using (var copy = image.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic))) - { - FillTensor(tensors[tensor_index].Tensor, copy, tensor_part_index, options, pixToTensor); - tensors[tensor_index].Offsets[tensor_part_index] = new OffsetBox(0, 0, image.Width, image.Height); - } - tensor_part_index++; - } - for (var x = 0; x < image.Width - xs; x += xs) + // Размеры для ресайза изображения к размеру по которому удобно идти кропами + var resizedForCropWidth = (int)Math.Round(Sw * resizedForCropWidthKoef, MidpointRounding.ToEven); + var resizedForCropHeight = (int)Math.Round(Sh * resizedForCropHeightKoef, MidpointRounding.ToEven); + + // Размеры кропа, равны входу сети, а не (options.Crop.Width, options.Crop.Height), т.к. для оптимизации изображение будет предварительно отресайзено + var CRw = options.InputWidth; + var CRh = options.InputHeight; + + // Расчет сдвигов между кропами + var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw; + var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh; + + using (var source = image.Clone(img => img.Resize(resizedForCropWidth, resizedForCropHeight, KnownResamplers.Bicubic))) { - var startx = x; - var dx = (x + options.Crop.Width) - image.Width; - if (dx > 0) - { - startx -= dx; - } - for (var y = 0; y < image.Height - ys; y += ys) + // Количество тензоров всего, во всех батчах суммарно + var count = CalculateFragmentsCount(source, options); + + // Проверка, укладывается ли количество тензоров поровну в батчи + int offset = count % options.MaxBatchSize; + + // Количество батчей + int count_tensor_batches = count / options.MaxBatchSize + (offset == 0 ? 0 : 1); + + // Батчи + var tensors = new ImagePredictionInput[count_tensor_batches]; + + // Инициализация батчей + Parallel.For(0, count_tensor_batches, batch_index => { - if (tensor_part_index > 0 && tensor_part_index % options.MaxBatchSize == 0) + if (batch_index < count_tensor_batches - 1) { - tensor_index++; - tensor_part_index = 0; + tensors[batch_index] = new ImagePredictionInput + { + Tensor = InitInputTensor(options, options.MaxBatchSize), + Offsets = new OffsetBox[options.MaxBatchSize], + Count = options.MaxBatchSize + }; } - var starty = y; - var dy = (y + options.Crop.Height) - image.Height; - if (dy > 0) + else { - starty -= dy; + tensors[batch_index] = new ImagePredictionInput + { + Tensor = InitInputTensor(options, offset == 0 ? options.MaxBatchSize : offset), + Offsets = new OffsetBox[offset == 0 ? options.MaxBatchSize : offset], + Count = offset == 0 ? options.MaxBatchSize : offset + }; } - using (var copy = image - .Clone(img => img - .Crop(new Rectangle(startx, starty, options.Crop.Width, options.Crop.Height)) - .Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic))) + }); + + // Заполнение батчей + int tensor_index = 0; + + // Если используется ресайз оригинала кроме кропов, пишется в первый батч в первый тензор + if (options.Crop.SaveOriginal) + { + using (var copy = source.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic))) { - FillTensor(tensors[tensor_index].Tensor, copy, tensor_part_index, options, pixToTensor); - tensors[tensor_index].Offsets[tensor_part_index] = new OffsetBox(startx, starty, options.Crop.Width, options.Crop.Height); + FillTensor(tensors[0].Tensor, copy, 0, options, pixToTensor); + tensors[tensor_index].Offsets[0] = new OffsetBox(0, 0, image.Width, image.Height); } - tensor_part_index++; + tensor_index++; } + tensor_index--; + Parallel.ForEach(SteppedIterator(0, source.Width, Dx), x => + { + // Можно запараллелить и тут, но выигрыш дает малоощутимый + for (int y = 0; y < source.Height; y += Dy) + { + var current_index = Interlocked.Increment(ref tensor_index); + // Индекс тензора внутри батча + var b_index = current_index % options.MaxBatchSize; + // Индекс батча + var p_index = (int)Math.Round((double)current_index / (double)options.MaxBatchSize, MidpointRounding.ToNegativeInfinity); + int w = CRw; + if ((x + CRw) > source.Width) + { + w = source.Width - x; + } + int h = CRh; + if ((y + CRh) > source.Height) + { + h = source.Height - y; + } + // Заполнение b_index тензора в p_index батче + FillTensor(tensors[p_index].Tensor, source, x, y, w, h, b_index, options, pixToTensor); + // Указание смещений для данного тензора + tensors[p_index].Offsets[b_index] = new OffsetBox(x, y, options.Crop.Width, options.Crop.Height); + } + }); + return tensors; } - return tensors; } - // if resize only - result = new PredictionInput[1]; + result = new ImagePredictionInput[1]; using (var copy = image.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic))) { Tensor tensor = InitInputTensor(options); FillTensor(tensor, copy, 0, options, pixToTensor); - result[0] = new PredictionInput { Count = 1, Offsets = null, Tensor = tensor }; + result[0] = new ImagePredictionInput + { + Count = 1, + Offsets = null, + Tensor = tensor + }; } return result; } + + private static IEnumerable SteppedIterator(int startIndex, int endIndex, int stepSize) + { + for (int i = startIndex; i < endIndex; i += stepSize) + { + yield return i; + } + } + public static Image Crop(Image source, float x1, float y1, float x2, float y2) { int left = 0; diff --git a/ZeroLevel.NN/Services/NMS.cs b/ZeroLevel.NN/Services/NMS.cs new file mode 100644 index 0000000..c9ccee5 --- /dev/null +++ b/ZeroLevel.NN/Services/NMS.cs @@ -0,0 +1,118 @@ +using ZeroLevel.NN.Models; + +namespace ZeroLevel.NN.Services +{ + public static class NMS + { + private const float IOU_THRESHOLD = .2f; + private const float IOU_MERGE_THRESHOLD = .5f; + + public static void Apply(List boxes) + { + for (int i = 0; i < boxes.Count - 1; i++) + { + var left = boxes[i]; + for (int j = i + 1; j < boxes.Count; j++) + { + var right = boxes[j]; + if (left.Class == right.Class) + { + // удаление вложенных боксов + var ni = NestingIndex(left, right); + if (ni == 1) + { + boxes.RemoveAt(i); + i--; + break; + } + else if (ni == 2) + { + boxes.RemoveAt(j); + j--; + } + // ----------------------------- + else + { + var iou = IOU(left, right); + if (iou > IOU_THRESHOLD) + { + if (left.Score > right.Score) + { + boxes.RemoveAt(j); + j--; + } + else + { + boxes.RemoveAt(i); + i--; + break; + } + } + /*else if (threshold > 0.01f && iou > 0.2f) + { + // UNION + var x1 = Math.Min(left.X, right.X); + var y1 = Math.Min(left.Y, right.Y); + var x2 = Math.Max(left.X + left.W, right.X + right.W); + var y2 = Math.Max(left.Y + left.H, right.Y + right.H); + var w = x2 - x1; + var h = y2 - y1; + boxes.RemoveAt(j); + boxes.RemoveAt(i); + boxes.Add(new Prediction + { + Class = left.Class, + Label = left.Label, + Score = Math.Max(left.Score, right.Score), + Cx = x1 + w / 2.0f, + Cy = y1 + h / 2.0f, + W = w, + H = h + }); + i--; + break; + }*/ + } + } + } + } + } + + /// + /// проверка на вложенность боксов + /// + static int NestingIndex(YoloPrediction box1, YoloPrediction box2) + { + if (box1.X > box2.X && + box1.Y > box2.Y && + (box1.X + box1.W) < (box2.X + box2.W) && + (box1.Y + box1.H) < (box2.Y + box2.H)) + { + return 1; + } + if (box2.X > box1.X && + box2.Y > box1.Y && + (box2.X + box2.W) < (box1.X + box1.W) && + (box2.Y + box2.H) < (box1.Y + box1.H)) + { + return 2; + } + return 0; + } + + static float IOU(YoloPrediction box1, YoloPrediction box2) + { + var left = (float)Math.Max(box1[0], box2[0]); + var right = (float)Math.Min(box1[2], box2[2]); + + var top = (float)Math.Max(box1[1], box2[1]); + var bottom = (float)Math.Min(box1[3], box2[3]); + + var width = right - left; + var height = bottom - top; + var intersectionArea = width * height; + + return intersectionArea / (float)(box1.Area + box2.Area - intersectionArea); + } + } +} diff --git a/ZeroLevel.NN/Services/SSDNN.cs b/ZeroLevel.NN/Services/SSDNN.cs index 87b65b1..484ae98 100644 --- a/ZeroLevel.NN/Services/SSDNN.cs +++ b/ZeroLevel.NN/Services/SSDNN.cs @@ -1,7 +1,6 @@ using Microsoft.ML.OnnxRuntime; using Microsoft.ML.OnnxRuntime.Tensors; using SixLabors.ImageSharp; -using SixLabors.ImageSharp.PixelFormats; using ZeroLevel.NN.Models; namespace ZeroLevel.NN @@ -18,6 +17,7 @@ namespace ZeroLevel.NN try { var so = SessionOptions.MakeSessionOptionWithCudaProvider(0); + so.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_VERBOSE; so.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL; _session = new InferenceSession(modelPath, so); } @@ -64,7 +64,7 @@ namespace ZeroLevel.NN vector[i] *= inverseLength; } } - protected PredictionInput[] MakeInputBatch(Image image, ImagePreprocessorOptions options) + protected ImagePredictionInput[] MakeInputBatch(Image image, ImagePreprocessorOptions options) { return ImagePreprocessor.ToTensors(image, options); } @@ -75,6 +75,22 @@ namespace ZeroLevel.NN return input[0].Tensor; } + protected int Argmax(float[] embedding) + { + if (embedding.Length == 0) return -1; + var im = 0; + var max = embedding[0]; + for (var i = 1; i < embedding.Length; i++) + { + if (embedding[i] > max) + { + im = i; + max = embedding[i]; + } + } + return im; + } + public void Dispose() { _session?.Dispose(); diff --git a/ZeroLevel.NN/ZeroLevel.NN.csproj b/ZeroLevel.NN/ZeroLevel.NN.csproj index 055fdc2..f6bbabf 100644 --- a/ZeroLevel.NN/ZeroLevel.NN.csproj +++ b/ZeroLevel.NN/ZeroLevel.NN.csproj @@ -7,7 +7,7 @@ True embedded none - 1.0.0.1 + 1.0.0.2 Ogoun Ogoun Copyright Ogoun 2022 @@ -15,13 +15,18 @@ https://github.com/ogoun/Zero/wiki https://github.com/ogoun/Zero git - New architectures + + AnyCPU;x64 False + + False + + True @@ -30,10 +35,10 @@ - - - - + + + + diff --git a/ZeroLevel.SqLite/BaseSqLiteDB.cs b/ZeroLevel.SqLite/BaseSqLiteDB.cs index ebb8b2a..a59174e 100644 --- a/ZeroLevel.SqLite/BaseSqLiteDB.cs +++ b/ZeroLevel.SqLite/BaseSqLiteDB.cs @@ -1,77 +1,94 @@ -using System; -using System.Data.SQLite; +using SQLite; +using System; +using System.Collections.Generic; using System.IO; +using System.Linq.Expressions; using ZeroLevel.Services.FileSystem; namespace ZeroLevel.SqLite { - public abstract class BaseSqLiteDB + public abstract class BaseSqLiteDB + : IDisposable + where T : class, new() { - #region Helpers - protected static bool HasColumn(SQLiteDataReader dr, string columnName) + protected SQLiteConnection _db; + public BaseSqLiteDB(string name) { - for (int i = 0; i < dr.FieldCount; i++) - { - if (dr.GetName(i).Equals(columnName, StringComparison.InvariantCultureIgnoreCase)) - return true; - } - return false; + _db = new SQLiteConnection(PrepareDb(name)); } - protected static Tr Read(SQLiteDataReader reader, int index) + + public T Append(T record) { - if (reader == null || reader.FieldCount <= index || reader[index] == DBNull.Value) return default; - Type t; - if ((t = Nullable.GetUnderlyingType(typeof(Tr))) != null) - { - return (Tr)Convert.ChangeType(reader[index], t); - } - return (Tr)Convert.ChangeType(reader[index], typeof(Tr)); + _db.Insert(record); + return record; } - protected static Tr Read(SQLiteDataReader reader, string name) + + public void CreateTable() { - if (reader == null || !HasColumn(reader, name) || reader[name] == DBNull.Value) return default; - Type t; - if ((t = Nullable.GetUnderlyingType(typeof(Tr))) != null) - { - return (Tr)Convert.ChangeType(reader[name], t); - } - return (Tr)Convert.ChangeType(reader[name], typeof(Tr)); + _db.CreateTable(); } - protected static void Execute(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null) + public void DropTable() { - using (var cmd = new SQLiteCommand(query, connection)) - { - if (parameters != null && parameters.Length > 0) - { - cmd.Parameters.AddRange(parameters); - } - cmd.ExecuteNonQuery(); - } + _db.DropTable(); } - protected static object ExecuteScalar(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null) + public IEnumerable SelectAll() { - using (var cmd = new SQLiteCommand(query, connection)) + return _db.Table(); + } + + public IEnumerable SelectBy(Expression> predicate) + { + return _db.Table().Where(predicate); + } + + public T Single(Expression> predicate) + { + return _db.Table().FirstOrDefault(predicate); + } + + public T Single(Expression> predicate, Expression> orderBy, bool desc = false) + { + if (desc) { - if (parameters != null && parameters.Length > 0) - { - cmd.Parameters.AddRange(parameters); - } - return cmd.ExecuteScalar(); + return _db.Table().Where(predicate).OrderByDescending(orderBy).FirstOrDefault(); } + return _db.Table().Where(predicate).OrderBy(orderBy).FirstOrDefault(); } - protected static SQLiteDataReader Read(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null) + public T Single(Expression> orderBy, bool desc = false) { - using (var cmd = new SQLiteCommand(query, connection)) + if (desc) { - if (parameters != null && parameters.Length > 0) - { - cmd.Parameters.AddRange(parameters); - } - return cmd.ExecuteReader(); + return _db.Table().OrderByDescending(orderBy).FirstOrDefault(); } + return _db.Table().OrderBy(orderBy).FirstOrDefault(); + } + + public IEnumerable SelectBy(int N, Expression> predicate) + { + return _db.Table().Where(predicate).Take(N); + } + + public long Count() + { + return _db.Table().Count(); + } + + public long Count(Expression> predicate) + { + return _db.Table().Count(predicate); + } + + public int Delete(Expression> predicate) + { + return _db.Table().Delete(predicate); + } + + public void Update(T record) + { + _db.Update(record); } protected static string PrepareDb(string path) @@ -80,13 +97,23 @@ namespace ZeroLevel.SqLite { path = Path.Combine(FSUtils.GetAppLocalDbDirectory(), path); } - if (!File.Exists(path)) - { - SQLiteConnection.CreateFile(path); - } return Path.GetFullPath(path); } - #endregion Helpers + protected abstract void DisposeStorageData(); + + public void Dispose() + { + DisposeStorageData(); + try + { + _db?.Close(); + _db?.Dispose(); + } + catch (Exception ex) + { + Log.Error(ex, "[BaseSqLiteDB] Fault close db connection"); + } + } } } diff --git a/ZeroLevel.SqLite/Model/User.cs b/ZeroLevel.SqLite/Model/User.cs index f944657..fd75e96 100644 --- a/ZeroLevel.SqLite/Model/User.cs +++ b/ZeroLevel.SqLite/Model/User.cs @@ -1,10 +1,15 @@ -namespace ZeroLevel.SqLite +using SQLite; + +namespace ZeroLevel.SqLite { public class User { + [PrimaryKey, AutoIncrement] public long Id { get; set; } + [Indexed] public string UserName { get; set; } public string DisplayName { get; set; } + [Indexed] public byte[] PasswordHash { get; set; } public long Timestamp { get; set; } public long Creator { get; set; } diff --git a/ZeroLevel.SqLite/SqLiteDelayDataStorage.cs b/ZeroLevel.SqLite/SqLiteDelayDataStorage.cs index 943dc6c..301b7f6 100644 --- a/ZeroLevel.SqLite/SqLiteDelayDataStorage.cs +++ b/ZeroLevel.SqLite/SqLiteDelayDataStorage.cs @@ -1,24 +1,28 @@ -using System; -using System.Collections.Generic; -using System.Data.SQLite; -using System.IO; -using System.Text; +using SQLite; +using System; using System.Threading; using ZeroLevel.Services.Serialization; using ZeroLevel.Services.Shedulling; namespace ZeroLevel.SqLite { + public sealed class ExpirationRecord + { + [PrimaryKey, AutoIncrement] + public long Id { get; set; } + [Indexed] + public long Expiration { get; set; } + public byte[] Data { get; set; } + } + public sealed class SqLiteDelayDataStorage - : BaseSqLiteDB, IDisposable - where T : IBinarySerializable + : BaseSqLiteDB + where T : class, IBinarySerializable, new() { #region Fields private readonly IExpirationSheduller _sheduller; private readonly Func _expire_date_calc_func; - private readonly SQLiteConnection _db; - private readonly string _table_name; private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim(); #endregion Fields @@ -28,14 +32,10 @@ namespace ZeroLevel.SqLite public SqLiteDelayDataStorage(string database_file_path, Func expire_callback, Func expire_date_calc_func) + : base(database_file_path) { this._expire_date_calc_func = expire_date_calc_func; - var path = PrepareDb(database_file_path); - _table_name = "expiration"; - _db = new SQLiteConnection($"Data Source={path};Version=3;"); - _db.Open(); - Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB, expirationtime INTEGER)", _db); - Execute($"CREATE INDEX IF NOT EXISTS expirationtime_index ON {_table_name} (expirationtime)", _db); + CreateTable(); _sheduller = Sheduller.CreateExpirationSheduller(); OnExpire += expire_callback; Preload(); @@ -64,13 +64,8 @@ namespace ZeroLevel.SqLite long id = -1; try { - Execute($"INSERT INTO {_table_name} ('body', 'expirationtime') values (@body, @expirationtime)", _db, - new SQLiteParameter[] - { - new SQLiteParameter("body", MessageSerializer.Serialize(packet)), - new SQLiteParameter("expirationtime", expirationTime) - }); - id = (long)ExecuteScalar("select last_insert_rowid();", _db); + var r = Append(new ExpirationRecord { Expiration = expirationTime, Data = MessageSerializer.Serialize(packet) }); + id = r.Id; } catch (Exception ex) { @@ -91,17 +86,13 @@ namespace ZeroLevel.SqLite private void Preload() { - SQLiteDataReader reader; _rwLock.EnterReadLock(); try { - reader = Read($"SELECT id, expirationtime FROM {_table_name}", _db); - while (reader.Read()) + foreach (var record in SelectAll()) { - var id = reader.GetInt64(0); - _sheduller.Push(new DateTime(reader.GetInt64(1), DateTimeKind.Local), (k) => Pop(id)); + _sheduller.Push(new DateTime(record.Expiration, DateTimeKind.Local), (k) => Pop(record.Id)); } - reader.Close(); } catch (Exception ex) { @@ -111,7 +102,6 @@ namespace ZeroLevel.SqLite { _rwLock.ExitReadLock(); } - reader = null; } private void Pop(long id) @@ -122,7 +112,7 @@ namespace ZeroLevel.SqLite _rwLock.EnterReadLock(); try { - body = (byte[])ExecuteScalar($"SELECT body FROM {_table_name} WHERE id=@id", _db, new SQLiteParameter[] { new SQLiteParameter("id", id) }); + body = Single(r=>r.Id == id)?.Data; } catch (Exception ex) { @@ -161,8 +151,7 @@ namespace ZeroLevel.SqLite _rwLock.EnterWriteLock(); try { - Execute($"DELETE FROM {_table_name} WHERE id = @id", _db, - new SQLiteParameter[] { new SQLiteParameter("id", id) }); + Delete(r => r.Id == id); } catch (Exception ex) { @@ -178,17 +167,8 @@ namespace ZeroLevel.SqLite #region IDisposable - public void Dispose() + protected override void DisposeStorageData() { - try - { - _db?.Close(); - _db?.Dispose(); - } - catch (Exception ex) - { - Log.Error(ex, "[SqLiteDelayDataStorage] Fault close db connection"); - } _sheduller.Dispose(); } diff --git a/ZeroLevel.SqLite/SqLiteDupStorage.cs b/ZeroLevel.SqLite/SqLiteDupStorage.cs index 84d83ea..117754b 100644 --- a/ZeroLevel.SqLite/SqLiteDupStorage.cs +++ b/ZeroLevel.SqLite/SqLiteDupStorage.cs @@ -1,6 +1,6 @@ -using System; +using SQLite; +using System; using System.Collections.Generic; -using System.Data.SQLite; using System.Linq; using System.Security.Cryptography; using System.Text; @@ -8,40 +8,37 @@ using System.Threading; namespace ZeroLevel.SqLite { + public sealed class DuplicateRecord + { + [PrimaryKey, AutoIncrement] + public long Id { get; set; } + [Indexed] + public string Hash { get; set; } + public long Timestamp { get; set; } + public byte[] Data { get; set; } + } + /// /// Хранит данные указанное число дней, и позволяет выполнить проверку наличия данных, для отбрасывания дубликатов /// public sealed class SqLiteDupStorage - : BaseSqLiteDB, IDisposable + : BaseSqLiteDB { #region Fields - private const string DEFAUL_TABLE_NAME = "History"; - - private readonly SQLiteConnection _db; private readonly long _removeOldRecordsTaskKey; private readonly int _countDays; - private readonly string _table_name; private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim(); #endregion Fields #region Private members - - private sealed class DuplicationStorageRecord - { - public string Hash { get; set; } - public byte[] Body { get; set; } - public long Timestamp { get; set; } - } - private void RemoveOldRecordsTask(long key) { _rwLock.EnterWriteLock(); try { - Execute($"DELETE FROM {_table_name} WHERE timestamp < @limit", _db, - new SQLiteParameter[] { new SQLiteParameter("limit", DateTime.Now.AddDays(-_countDays).Ticks) }); + Delete(r => r.Timestamp < DateTime.Now.AddDays(-_countDays).Ticks); } catch (Exception ex) { @@ -57,21 +54,10 @@ namespace ZeroLevel.SqLite #region Ctor - public SqLiteDupStorage(string database_file_path, string tableName, int period) + public SqLiteDupStorage(string database_file_path, int period) + : base(database_file_path) { - var path = PrepareDb(database_file_path); - if (string.IsNullOrWhiteSpace(tableName)) - { - _table_name = DEFAUL_TABLE_NAME; - } - else - { - _table_name = tableName; - } - _db = new SQLiteConnection($"Data Source={path};Version=3;"); - _db.Open(); - Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, body BLOB, timestamp INTEGER)", _db); - Execute($"CREATE INDEX IF NOT EXISTS hash_index ON {_table_name} (hash)", _db); + CreateTable(); _countDays = period > 0 ? period : 1; _removeOldRecordsTaskKey = Sheduller.RemindEvery(TimeSpan.FromMinutes(1), RemoveOldRecordsTask); } @@ -87,17 +73,14 @@ namespace ZeroLevel.SqLite { var hash = GenerateSHA256String(body); var timestamp = DateTime.Now.Ticks; - SQLiteDataReader reader; _rwLock.EnterReadLock(); var exists = new List(); try { - reader = Read($"SELECT body FROM {_table_name} WHERE hash=@hash", _db, new SQLiteParameter[] { new SQLiteParameter("hash", hash) }); - while (reader.Read()) + foreach (var record in SelectBy(r => r.Hash == hash)) { - exists.Add((byte[])reader.GetValue(0)); + exists.Add(record.Data); } - reader.Close(); } catch (Exception ex) { @@ -108,7 +91,6 @@ namespace ZeroLevel.SqLite { _rwLock.ExitReadLock(); } - reader = null; if (exists.Any()) { foreach (var candidate in exists) @@ -120,13 +102,12 @@ namespace ZeroLevel.SqLite _rwLock.EnterWriteLock(); try { - Execute($"INSERT INTO {_table_name} ('hash', 'body', 'timestamp') values (@hash, @body, @timestamp)", _db, - new SQLiteParameter[] - { - new SQLiteParameter("hash", hash), - new SQLiteParameter("body", body), - new SQLiteParameter("timestamp", timestamp) - }); + Append(new DuplicateRecord + { + Data = body, + Hash = hash, + Timestamp = timestamp + }); } catch (Exception ex) { @@ -143,17 +124,9 @@ namespace ZeroLevel.SqLite #region IDisposable - public void Dispose() + protected override void DisposeStorageData() { Sheduller.Remove(_removeOldRecordsTaskKey); - try - { - _db?.Dispose(); - } - catch (Exception ex) - { - Log.Error(ex, "[SQLiteDupStorage] Fault close db connection"); - } } #endregion IDisposable diff --git a/ZeroLevel.SqLite/SqLitePacketBuffer.cs b/ZeroLevel.SqLite/SqLitePacketBuffer.cs index e20da5d..d17de22 100644 --- a/ZeroLevel.SqLite/SqLitePacketBuffer.cs +++ b/ZeroLevel.SqLite/SqLitePacketBuffer.cs @@ -1,15 +1,25 @@ -using System; -using System.Data.SQLite; +using SQLite; +using System; using System.Threading; using ZeroLevel.Services.Serialization; namespace ZeroLevel.SqLite { + + public sealed class PacketRecord + { + [PrimaryKey, AutoIncrement] + public long Id { get; set; } + [Indexed] + public long Timestamp { get; set; } + public byte[] Data { get; set; } + } + /// /// Промежуточное/временное хранилище пакетов данных, для случаев сбоя доставок через шину данных /// public sealed class SqLitePacketBuffer - : BaseSqLiteDB, IDisposable + : BaseSqLiteDB where T : IBinarySerializable { private sealed class PacketBufferRecord @@ -20,20 +30,14 @@ namespace ZeroLevel.SqLite #region Fields - private readonly SQLiteConnection _db; - private readonly string _table_name; private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim(); #endregion Fields public SqLitePacketBuffer(string database_file_path) + : base(database_file_path) { - var path = PrepareDb(database_file_path); - _table_name = "packets"; - _db = new SQLiteConnection($"Data Source={path};Version=3;"); - _db.Open(); - Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB, created INTEGER)", _db); - Execute($"CREATE INDEX IF NOT EXISTS created_index ON {_table_name} (created)", _db); + CreateTable(); } public void Push(T frame) @@ -43,13 +47,11 @@ namespace ZeroLevel.SqLite var creationTime = DateTime.Now.Ticks; try { - Execute($"INSERT INTO {_table_name} ('body', 'created') values (@body, @created)", _db, - new SQLiteParameter[] - { - new SQLiteParameter("body", MessageSerializer.Serialize(frame)), - new SQLiteParameter("created", creationTime) - }); - id = (long)ExecuteScalar("select last_insert_rowid();", _db); + id = Append(new PacketRecord + { + Data = MessageSerializer.Serialize(frame), + Timestamp = creationTime + }).Id; } catch (Exception ex) { @@ -65,25 +67,20 @@ namespace ZeroLevel.SqLite { bool success = false; long id = -1; - SQLiteDataReader reader; _rwLock.EnterReadLock(); try { - reader = Read($"SELECT id, body FROM {_table_name} ORDER BY created ASC LIMIT 1", _db); - if (reader.Read()) + var record = Single(r => r.Timestamp); + id = record.Id; + var body = record.Data; + try + { + success = pop_callback(MessageSerializer.Deserialize(body)); + } + catch (Exception ex) { - id = reader.GetInt64(0); - var body = (byte[])reader.GetValue(1); - try - { - success = pop_callback(MessageSerializer.Deserialize(body)); - } - catch (Exception ex) - { - Log.Error(ex, "Fault handle buffered data"); - } + Log.Error(ex, "Fault handle buffered data"); } - reader.Close(); } catch (Exception ex) { @@ -97,7 +94,6 @@ namespace ZeroLevel.SqLite { RemoveRecordById(id); } - reader = null; return success; } @@ -106,8 +102,7 @@ namespace ZeroLevel.SqLite _rwLock.EnterWriteLock(); try { - Execute($"DELETE FROM {_table_name} WHERE id = @id", _db, - new SQLiteParameter[] { new SQLiteParameter("id", id) }); + Delete(r => r.Id == id); } catch (Exception ex) { @@ -119,17 +114,8 @@ namespace ZeroLevel.SqLite } } - public void Dispose() + protected override void DisposeStorageData() { - try - { - _db?.Close(); - _db?.Dispose(); - } - catch (Exception ex) - { - Log.Error(ex, "[SqLitePacketBuffer] Fault close db connection"); - } } } } diff --git a/ZeroLevel.SqLite/SqLiteUserRepository.cs b/ZeroLevel.SqLite/SqLiteUserRepository.cs index 168a3a4..d39a706 100644 --- a/ZeroLevel.SqLite/SqLiteUserRepository.cs +++ b/ZeroLevel.SqLite/SqLiteUserRepository.cs @@ -1,18 +1,15 @@ using System; using System.Collections.Generic; -using System.Data.SQLite; using System.Threading; using ZeroLevel.Models; namespace ZeroLevel.SqLite { public class SqLiteUserRepository - : BaseSqLiteDB + : BaseSqLiteDB { #region Fields - private readonly SQLiteConnection _db; - private readonly string _table_name = "users"; private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim(); #endregion Fields @@ -20,13 +17,8 @@ namespace ZeroLevel.SqLite #region Ctor public SqLiteUserRepository() + : base("users.db") { - var path =PrepareDb("users.db"); - _db = new SQLiteConnection($"Data Source={path};Version=3;"); - _db.Open(); - Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT, displayname TEXT, hash BLOB, timestamp INTEGER, creator INTEGER, role INTEGER)", _db); - Execute($"CREATE INDEX IF NOT EXISTS username_index ON {_table_name} (username)", _db); - Execute($"CREATE INDEX IF NOT EXISTS hash_index ON {_table_name} (hash)", _db); } #endregion Ctor @@ -34,25 +26,13 @@ namespace ZeroLevel.SqLite public IEnumerable GetAll() { var list = new List(); - SQLiteDataReader reader; _rwLock.EnterReadLock(); try { - reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name}", _db); - while (reader.Read()) + foreach (var r in SelectAll()) { - list.Add(new User - { - Id = reader.GetInt64(0), - UserName = reader.GetString(1), - DisplayName = Read(reader, 2), - PasswordHash = (byte[])reader.GetValue(3), - Timestamp = reader.GetInt64(4), - Creator = reader.GetInt64(5), - Role = (UserRole)reader.GetInt32(6) - }); + list.Add(r); } - reader.Close(); } catch (Exception ex) { @@ -62,34 +42,16 @@ namespace ZeroLevel.SqLite { _rwLock.ExitReadLock(); } - reader = null; return list; } public User Get(long id) { User user = null; - SQLiteDataReader reader; _rwLock.EnterReadLock(); try { - reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name} WHERE id = @id", _db, - new SQLiteParameter[] { new SQLiteParameter("id", id) }); - if (reader.Read()) - { - var body = (byte[])reader.GetValue(1); - user = new User - { - Id = reader.GetInt64(0), - UserName = reader.GetString(1), - DisplayName = reader.GetString(2), - PasswordHash = (byte[])reader.GetValue(3), - Timestamp = reader.GetInt64(4), - Creator = reader.GetInt64(5), - Role = (UserRole)reader.GetInt32(6) - }; - } - reader.Close(); + user = Single(r => r.Id == id); } catch (Exception ex) { @@ -99,37 +61,16 @@ namespace ZeroLevel.SqLite { _rwLock.ExitReadLock(); } - reader = null; return user; } public User Get(string username, byte[] hash) { User user = null; - SQLiteDataReader reader; _rwLock.EnterReadLock(); try { - reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name} WHERE username = @username AND hash = @hash", _db, - new SQLiteParameter[] - { - new SQLiteParameter("username", username), - new SQLiteParameter("hash", hash) - }); - if (reader.Read()) - { - user = new User - { - Id = reader.GetInt64(0), - UserName = reader.GetString(1), - DisplayName = reader.GetString(2), - PasswordHash = (byte[])reader.GetValue(3), - Timestamp = reader.GetInt64(4), - Creator = reader.GetInt64(5), - Role = (UserRole)reader.GetInt32(6) - }; - } - reader.Close(); + user = Single(r => r.UserName == username && r.PasswordHash == hash); } catch (Exception ex) { @@ -139,7 +80,6 @@ namespace ZeroLevel.SqLite { _rwLock.ExitReadLock(); } - reader = null; return user; } @@ -150,22 +90,12 @@ namespace ZeroLevel.SqLite var creationTime = DateTime.UtcNow.Ticks; try { - var count_obj = ExecuteScalar($"SELECT COUNT(*) FROM {_table_name} WHERE username=@username", _db, new SQLiteParameter[] { new SQLiteParameter("username", user.UserName) }); - if (count_obj != null && (long)count_obj > 0) + var count_obj = Count(r => r.UserName == user.UserName); + if (count_obj > 0) { - return InvokeResult.Fault("Пользователь уже существует"); + return InvokeResult.Fault("Пользователь с таким именем уже существует"); } - Execute($"INSERT INTO {_table_name} ('username', 'displayname', 'hash', 'timestamp', 'creator', 'role') values (@username, @displayname, @hash, @timestamp, @creator, @role)", _db, - new SQLiteParameter[] - { - new SQLiteParameter("username", user.UserName), - new SQLiteParameter("displayname", user.DisplayName), - new SQLiteParameter("hash", user.PasswordHash), - new SQLiteParameter("timestamp", creationTime), - new SQLiteParameter("creator", user.Creator), - new SQLiteParameter("role", user.Role) - }); - id = (long)ExecuteScalar("select last_insert_rowid();", _db); + id = Append(user).Id; } catch (Exception ex) { @@ -184,8 +114,7 @@ namespace ZeroLevel.SqLite _rwLock.EnterWriteLock(); try { - Execute($"DELETE FROM {_table_name} WHERE username = @username", _db, - new SQLiteParameter[] { new SQLiteParameter("username", login) }); + Delete(r => r.UserName == login); return InvokeResult.Succeeding(); } catch (Exception ex) @@ -198,5 +127,9 @@ namespace ZeroLevel.SqLite _rwLock.ExitWriteLock(); } } + + protected override void DisposeStorageData() + { + } } } diff --git a/ZeroLevel.SqLite/UserCacheRepository.cs b/ZeroLevel.SqLite/UserCacheRepository.cs index f48c2b4..569bc90 100644 --- a/ZeroLevel.SqLite/UserCacheRepository.cs +++ b/ZeroLevel.SqLite/UserCacheRepository.cs @@ -1,36 +1,36 @@ -using System; +using SQLite; +using System; using System.Collections.Generic; -using System.Data.SQLite; using System.Runtime.CompilerServices; using System.Threading; using ZeroLevel.Services.Serialization; namespace ZeroLevel.SqLite { + public sealed class DataRecord + { + [PrimaryKey, AutoIncrement] + public long Id { get; set; } + [Indexed] + public string Key { get; set; } + public byte[] Data { get; set; } + } public sealed class UserCacheRepository - : BaseSqLiteDB, IDisposable + : BaseSqLiteDB where T : IBinarySerializable { #region Fields - private readonly SQLiteConnection _db; private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim(); - private readonly string _tableName; #endregion Fields #region Ctor public UserCacheRepository() + : base(typeof(T).Name) { - _tableName = typeof(T).Name; - - var path = PrepareDb($"{_tableName}_user_cachee.db"); - _db = new SQLiteConnection($"Data Source={path};Version=3;"); - _db.Open(); - - Execute($"CREATE TABLE IF NOT EXISTS {_tableName} (id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT, body BLOB)", _db); - Execute($"CREATE INDEX IF NOT EXISTS key_index ON {_tableName} (key)", _db); + CreateTable(); } #endregion Ctor @@ -47,8 +47,8 @@ namespace ZeroLevel.SqLite _rwLock.EnterReadLock(); try { - var count_obj = ExecuteScalar($"SELECT COUNT(*) FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[] { new SQLiteParameter("key", key) }); - if (count_obj != null && (long)count_obj > 0) + var count_obj = Count(r => r.Key == key); + if (count_obj > 0) { update = true; } @@ -68,20 +68,16 @@ namespace ZeroLevel.SqLite var body = MessageSerializer.Serialize(data); if (update) { - Execute($"UPDATE {_tableName} SET body=@body WHERE key=@key", _db, - new SQLiteParameter[] - { - new SQLiteParameter("key", key), - new SQLiteParameter("body", body) - }); + var r = Single(r => r.Key == key); + r.Data = body; + Update(r); } else { - Execute($"INSERT INTO {_tableName} ('key', 'body') values (@key, @body)", _db, - new SQLiteParameter[] + Append(new DataRecord { - new SQLiteParameter("key", key), - new SQLiteParameter("body", body) + Data = body, + Key = key }); } return true; @@ -103,10 +99,7 @@ namespace ZeroLevel.SqLite _rwLock.EnterWriteLock(); try { - Execute($"DELETE FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[] - { - new SQLiteParameter("key", key) - }); + Delete(r => r.Key == key); } catch (Exception ex) { @@ -124,10 +117,7 @@ namespace ZeroLevel.SqLite _rwLock.EnterWriteLock(); try { - return Convert.ToInt64(ExecuteScalar($"SELECT COUNT(*) FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[] - { - new SQLiteParameter("key", key) - })); + return Count(r => r.Key == key); } catch (Exception ex) { @@ -144,23 +134,17 @@ namespace ZeroLevel.SqLite { var key = KEY(userid, name); var result = new List(); - SQLiteDataReader reader; _rwLock.EnterReadLock(); try { - reader = Read($"SELECT [body] FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[] + foreach (var r in SelectBy(r=>r.Key == key)) { - new SQLiteParameter("key", key) - }); - while (reader.Read()) - { - var data = Read(reader, 0); + var data = r.Data; if (data != null) { result.Add(MessageSerializer.Deserialize(data)); } } - reader.Close(); } catch (Exception ex) { @@ -169,7 +153,6 @@ namespace ZeroLevel.SqLite finally { _rwLock.ExitReadLock(); - reader = null; } return result; } @@ -177,17 +160,9 @@ namespace ZeroLevel.SqLite #endregion API #region IDisposable - - public void Dispose() + + protected override void DisposeStorageData() { - try - { - _db?.Dispose(); - } - catch (Exception ex) - { - Log.Error(ex, "[UserCacheRepository] Fault close db connection"); - } } #endregion IDisposable diff --git a/ZeroLevel.SqLite/ZeroLevel.SqLite.csproj b/ZeroLevel.SqLite/ZeroLevel.SqLite.csproj index c816c7c..3f601f3 100644 --- a/ZeroLevel.SqLite/ZeroLevel.SqLite.csproj +++ b/ZeroLevel.SqLite/ZeroLevel.SqLite.csproj @@ -13,9 +13,9 @@ Based on System.Data.SQLite.Core AnyCPU;x64;x86 Move to new ZeroLevel version - 1.0.3.0 - 1.0.3.0 - 1.0.3.0 + 1.0.4.0 + 1.0.4.0 + 1.0.4.0 Library @@ -27,11 +27,17 @@ Based on System.Data.SQLite.Core - + + + + PreserveNewest + + + diff --git a/ZeroLevel.SqLite/sqlite3.dll b/ZeroLevel.SqLite/sqlite3.dll new file mode 100644 index 0000000..70a8f7f Binary files /dev/null and b/ZeroLevel.SqLite/sqlite3.dll differ diff --git a/ZeroLevel.UnitTests/SuffixAutomataTests.cs b/ZeroLevel.UnitTests/SuffixAutomataTests.cs new file mode 100644 index 0000000..8d20a38 --- /dev/null +++ b/ZeroLevel.UnitTests/SuffixAutomataTests.cs @@ -0,0 +1,42 @@ +using Xunit; +using ZeroLevel.Semantic; +using ZeroLevel.Services.Text; + +namespace ZeroLevel.UnitTests +{ + public class SuffixAutomataTests + { + [Fact] + public void IsSubstringTest() + { + var text = @"=Однако эту оценку легко показать и без знания алгоритма. Вспомним о том, что число состояний равно количеству различных значений множеств endpos.#"; + var automata = new SuffixAutomata(); + automata.Init(); + foreach (var ch in text) + { + automata.Extend(ch); + } + Assert.True(automata.IsSubstring("Вспомним")); + Assert.True(automata.IsSubstring("")); // empty line + Assert.True(automata.IsSubstring("#")); // end line + Assert.True(automata.IsSubstring("=")); // start line + Assert.True(automata.IsSubstring(null)); // null + Assert.False(automata.IsSubstring("равноценно")); + Assert.False(automata.IsSubstring("нетслова")); + } + + [Fact] + public void IntersectionTest() + { + var text = @"=Однако эту оценку легко показать и без знания алгоритма. Вспомним о том, что число состояний равно количеству различных значений множеств endpos.#"; + var i = LongestCommonSubstring.LCS(text, "енк"); + Assert.Equal("енк", i); + + i = LongestCommonSubstring.LCS(text, "стоя"); + Assert.Equal("стоя", i); + + i = LongestCommonSubstring.LCS(text, "горизонт"); + Assert.Equal("гори", i); + } + } +} diff --git a/ZeroLevel.sln b/ZeroLevel.sln index 8d14039..229af50 100644 --- a/ZeroLevel.sln +++ b/ZeroLevel.sln @@ -39,8 +39,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "WPFExamples", "WPFExamples" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ConnectionTest", "ConnectionTest", "{D5207A5A-2F27-4992-9BA5-0BDCFC59F133}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Client", "ConnectionTest\Client\Client.csproj", "{08CDD42E-E324-40A4-88C3-EDD0493AAF84}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Server", "ConnectionTest\Server\Server.csproj", "{3496A688-0749-48C2-BD60-ABB42A5C17C9}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ZeroLevel.Qdrant", "ZeroLevel.Qdrant\ZeroLevel.Qdrant.csproj", "{7188B89E-96EB-4EFB-AAFB-D0A823031F99}" @@ -63,7 +61,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Source", "TestPipeLine\Sour EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Watcher", "TestPipeLine\Watcher\Watcher.csproj", "{F70842E7-9A1D-4CC4-9F55-0953AEC9C7C8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ZeroLevel.NN", "ZeroLevel.NN\ZeroLevel.NN.csproj", "{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ZeroLevel.NN", "ZeroLevel.NN\ZeroLevel.NN.csproj", "{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Client", "ConnectionTest\Client\Client.csproj", "{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AutoLoader", "AutoLoader", "{2EF83101-63BC-4397-A005-A747189143D4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AppContainer", "AutoLoader\AppContainer\AppContainer.csproj", "{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -171,18 +175,6 @@ Global {04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x64.Build.0 = Release|x64 {04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x86.ActiveCfg = Release|x86 {04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x86.Build.0 = Release|x86 - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|Any CPU.Build.0 = Debug|Any CPU - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x64.ActiveCfg = Debug|x64 - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x64.Build.0 = Debug|x64 - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x86.ActiveCfg = Debug|x86 - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x86.Build.0 = Debug|x86 - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|Any CPU.ActiveCfg = Release|Any CPU - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|Any CPU.Build.0 = Release|Any CPU - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x64.ActiveCfg = Release|x64 - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x64.Build.0 = Release|x64 - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x86.ActiveCfg = Release|x86 - {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x86.Build.0 = Release|x86 {3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|Any CPU.Build.0 = Debug|Any CPU {3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|x64.ActiveCfg = Debug|x64 @@ -327,12 +319,35 @@ Global {C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x64.Build.0 = Release|Any CPU {C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x86.ActiveCfg = Release|Any CPU {C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x86.Build.0 = Release|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x64.ActiveCfg = Debug|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x64.Build.0 = Debug|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x86.ActiveCfg = Debug|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x86.Build.0 = Debug|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|Any CPU.Build.0 = Release|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x64.ActiveCfg = Release|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x64.Build.0 = Release|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x86.ActiveCfg = Release|Any CPU + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x86.Build.0 = Release|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x64.ActiveCfg = Debug|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x64.Build.0 = Debug|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x86.ActiveCfg = Debug|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x86.Build.0 = Debug|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|Any CPU.Build.0 = Release|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x64.ActiveCfg = Release|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x64.Build.0 = Release|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x86.ActiveCfg = Release|Any CPU + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution - {08CDD42E-E324-40A4-88C3-EDD0493AAF84} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133} {3496A688-0749-48C2-BD60-ABB42A5C17C9} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133} {54BB2BA9-DAC0-4162-8BC0-E4A9B898CBB0} = {FC074553-5D9F-4DF1-9130-7092E37DE768} {3D0FE0BA-F7B1-4A63-BBA4-C96514A68426} = {FC074553-5D9F-4DF1-9130-7092E37DE768} @@ -340,6 +355,8 @@ Global {5CCFF557-C91F-4DD7-9530-D76FE517DA98} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15} {82202433-6426-4737-BAB2-473AC1F74C5D} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15} {F70842E7-9A1D-4CC4-9F55-0953AEC9C7C8} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15} + {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133} + {9DE345EA-955B-41A8-93AF-277C0B5A9AC5} = {2EF83101-63BC-4397-A005-A747189143D4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A65DB16F-877D-4586-A9F3-8BBBFBAF5CEB} diff --git a/ZeroLevel/Services/Extensions/LinqExtension.cs b/ZeroLevel/Services/Extensions/LinqExtension.cs index 5a0a5af..704e08c 100644 --- a/ZeroLevel/Services/Extensions/LinqExtension.cs +++ b/ZeroLevel/Services/Extensions/LinqExtension.cs @@ -4,6 +4,21 @@ namespace System.Linq { public static class LinqExtension { + public static void ThrowIfNull(this T val, string message = null) + { + if (null == val) + throw new ArgumentNullException(message); + } + + public static void Apply(this IEnumerable seq, Action action) + { + seq.ThrowIfNull(); + action.ThrowIfNull(); + foreach (var e in seq) + { + action.Invoke(e); + } + } public static IEnumerable ZipLongest(this IEnumerable left, IEnumerable right) { IEnumerator leftEnumerator = left.GetEnumerator(); diff --git a/ZeroLevel/Services/FileSystem/FSUtils.cs b/ZeroLevel/Services/FileSystem/FSUtils.cs index 98f6e9a..3db7ff4 100644 --- a/ZeroLevel/Services/FileSystem/FSUtils.cs +++ b/ZeroLevel/Services/FileSystem/FSUtils.cs @@ -341,6 +341,27 @@ namespace ZeroLevel.Services.FileSystem var zipFile = Path.Combine(tmp.FullName, "zip.zip"); File.WriteAllBytes(zipFile, data); ZipFile.ExtractToDirectory(zipFile, targetFolder); + File.Delete(zipFile); + } + + public static void UnPackFolder(string zipFile, string targetFolder) + { + if (Directory.Exists(targetFolder)) + { + try + { + FSUtils.RemoveFolder(targetFolder, 3, 3000); + } + catch (Exception ex) + { + Log.SystemError(ex, $"[FSUtils] Fault clean folder '{Path.GetDirectoryName(targetFolder)}'"); + } + } + if (Directory.Exists(targetFolder) == false) + { + Directory.CreateDirectory(targetFolder); + } + ZipFile.ExtractToDirectory(zipFile, targetFolder); } public static void CopyDir(string sourceFolder, string targetFolder) diff --git a/ZeroLevel/Services/HashFunctions/StringHash.cs b/ZeroLevel/Services/HashFunctions/StringHash.cs index fb75a5c..9fddb0a 100644 --- a/ZeroLevel/Services/HashFunctions/StringHash.cs +++ b/ZeroLevel/Services/HashFunctions/StringHash.cs @@ -6,22 +6,23 @@ /// public static class StringHash { - public static uint DotNetFullHash(string str) + public static int DotNetFullHash(string str) { unchecked { int hash1 = (5381 << 16) + 5381; int hash2 = hash1; - - for (int i = 0; i < str.Length; i += 2) + if (str != null) { - hash1 = ((hash1 << 5) + hash1) ^ str[i]; - if (i == str.Length - 1) - break; - hash2 = ((hash2 << 5) + hash2) ^ str[i + 1]; + for (int i = 0; i < str.Length; i += 2) + { + hash1 = ((hash1 << 5) + hash1) ^ str[i]; + if (i == str.Length - 1) + break; + hash2 = ((hash2 << 5) + hash2) ^ str[i + 1]; + } } - - return (uint)(hash1 + (hash2 * 1566083941)) & 0x7FFFFFFF; + return (hash1 + (hash2 * 1566083941)) & 0x7FFFFFFF; } } diff --git a/ZeroLevel/Services/MemoryPools/JetPool.cs b/ZeroLevel/Services/MemoryPools/JetPool.cs index d49e5e0..411d344 100644 --- a/ZeroLevel/Services/MemoryPools/JetPool.cs +++ b/ZeroLevel/Services/MemoryPools/JetPool.cs @@ -15,4 +15,15 @@ namespace MemoryPools [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Return(T instance) => _freeObjectsQueue.Push(instance); } + + public class JetValPool + { + private readonly JetStack _freeObjectsQueue = new JetStack(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public T Get() => _freeObjectsQueue.Count > 0 ? _freeObjectsQueue.Pop() : default(T); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Return(T instance) => _freeObjectsQueue.Push(instance); + } } diff --git a/ZeroLevel/Services/Network/Contracts/ISocketClient.cs b/ZeroLevel/Services/Network/Contracts/ISocketClient.cs index 5cee702..a2c0480 100644 --- a/ZeroLevel/Services/Network/Contracts/ISocketClient.cs +++ b/ZeroLevel/Services/Network/Contracts/ISocketClient.cs @@ -13,8 +13,8 @@ namespace ZeroLevel.Network IRouter Router { get; } - bool Send(Frame data); - bool Request(Frame data, Action callback, Action fail = null); - bool Response(byte[] data, int identity); + void Send(Frame data); + void Request(Frame data, Action callback, Action fail = null); + void Response(byte[] data, int identity); } } diff --git a/ZeroLevel/Services/Network/ExClient.cs b/ZeroLevel/Services/Network/ExClient.cs index 81bc9cf..08ce2e3 100644 --- a/ZeroLevel/Services/Network/ExClient.cs +++ b/ZeroLevel/Services/Network/ExClient.cs @@ -22,7 +22,8 @@ namespace ZeroLevel.Network { try { - return _client.Send(FrameFactory.Create(inbox)); + _client.Send(FrameFactory.Create(inbox)); + return true; } catch (Exception ex) { @@ -35,7 +36,8 @@ namespace ZeroLevel.Network { try { - return _client.Send(FrameFactory.Create(inbox, data)); + _client.Send(FrameFactory.Create(inbox, data)); + return true; } catch (Exception ex) { @@ -47,8 +49,9 @@ namespace ZeroLevel.Network public bool Send(T message) { try - { - return _client.Send(FrameFactory.Create(BaseSocket.DEFAULT_MESSAGE_INBOX, MessageSerializer.SerializeCompatible(message))); + { + _client.Send(FrameFactory.Create(BaseSocket.DEFAULT_MESSAGE_INBOX, MessageSerializer.SerializeCompatible(message))); + return true; } catch (Exception ex) { @@ -60,8 +63,9 @@ namespace ZeroLevel.Network public bool Send(string inbox, T message) { try - { - return _client.Send(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible(message))); + { + _client.Send(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible(message))); + return true; } catch (Exception ex) { @@ -73,8 +77,9 @@ namespace ZeroLevel.Network public bool Request(string inbox, Action callback) { try - { - return _client.Request(FrameFactory.Create(inbox), f => callback(f)); + { + _client.Request(FrameFactory.Create(inbox), f => callback(f)); + return true; } catch (Exception ex) { @@ -87,7 +92,8 @@ namespace ZeroLevel.Network { try { - return _client.Request(FrameFactory.Create(inbox, data), f => callback(f)); + _client.Request(FrameFactory.Create(inbox, data), f => callback(f)); + return true; } catch (Exception ex) { @@ -100,7 +106,8 @@ namespace ZeroLevel.Network { try { - return _client.Request(FrameFactory.Create(inbox), f => callback(MessageSerializer.DeserializeCompatible(f))); + _client.Request(FrameFactory.Create(inbox), f => callback(MessageSerializer.DeserializeCompatible(f))); + return true; } catch (Exception ex) { @@ -113,7 +120,8 @@ namespace ZeroLevel.Network { try { - return _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_INBOX), f => callback(MessageSerializer.DeserializeCompatible(f))); + _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_INBOX), f => callback(MessageSerializer.DeserializeCompatible(f))); + return true; } catch (Exception ex) { @@ -126,8 +134,9 @@ namespace ZeroLevel.Network { try { - return _client.Request(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible(request)), + _client.Request(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible(request)), f => callback(MessageSerializer.DeserializeCompatible(f))); + return true; } catch (Exception ex) { @@ -140,8 +149,9 @@ namespace ZeroLevel.Network { try { - return _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_WITHOUT_ARGS_INBOX, MessageSerializer.SerializeCompatible(request)), + _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_WITHOUT_ARGS_INBOX, MessageSerializer.SerializeCompatible(request)), f => callback(MessageSerializer.DeserializeCompatible(f))); + return true; } catch (Exception ex) { diff --git a/ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs b/ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs index f64aec5..1fe8215 100644 --- a/ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs +++ b/ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs @@ -1,7 +1,6 @@ using MemoryPools; using System; using System.Threading; -using ZeroLevel.Services.Pools; namespace ZeroLevel.Network { diff --git a/ZeroLevel/Services/Network/SocketClient.cs b/ZeroLevel/Services/Network/SocketClient.cs index 367eac9..d41e32c 100644 --- a/ZeroLevel/Services/Network/SocketClient.cs +++ b/ZeroLevel/Services/Network/SocketClient.cs @@ -1,4 +1,5 @@ -using System; +using MemoryPools; +using System; using System.Collections.Concurrent; using System.Net; using System.Net.Sockets; @@ -19,8 +20,17 @@ namespace ZeroLevel.Network public int identity; public byte[] data; } + private struct OutcomingFrame + { + public bool is_request; + public int identity; + public byte[] data; + } - private BlockingCollection _incoming_queue = new BlockingCollection(); + private readonly JetValPool _outcomingFramesPool = new JetValPool(); + private ConcurrentQueue _incoming_queue = new ConcurrentQueue(); + private ConcurrentQueue _outcoming_queue = new ConcurrentQueue(); + private ManualResetEventSlim _outcomingFrameEvent = new ManualResetEventSlim(false); #endregion private Socket _clientSocket; @@ -31,6 +41,7 @@ namespace ZeroLevel.Network private readonly object _reconnection_lock = new object(); private long _heartbeat_key; private Thread _receiveThread; + private Thread _sendingThread; #endregion Private @@ -41,7 +52,6 @@ namespace ZeroLevel.Network try { _clientSocket = new Socket(ep.AddressFamily, SocketType.Stream, ProtocolType.Tcp); - _clientSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontLinger, true); _clientSocket.Connect(ep); OnConnect(this); } @@ -81,6 +91,11 @@ namespace ZeroLevel.Network _receiveThread.IsBackground = true; _receiveThread.Start(); + _sendingThread = new Thread(OutcomingFramesJob); + _sendingThread.IsBackground = true; + _sendingThread.Start(); + + _heartbeat_key = Sheduller.RemindEvery(TimeSpan.FromMilliseconds(MINIMUM_HEARTBEAT_UPDATE_PERIOD_MS), Heartbeat); } @@ -109,27 +124,26 @@ namespace ZeroLevel.Network public event Action OnDisconnect = (_) => { }; public IPEndPoint Endpoint { get; } - public bool Request(Frame frame, Action callback, Action fail = null) + public void Request(Frame frame, Action callback, Action fail = null) { - if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Request] Socket status: {Status}"); + if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Request] Socket status: {Status}"); var data = NetworkPacketFactory.Reqeust(MessageSerializer.Serialize(frame), out int id); _requests.RegisterForFrame(id, callback, fail); - return Send(id, true, data); + Send(id, true, data); } - public bool Send(Frame frame) + public void Send(Frame frame) { if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Send] Socket status: {Status}"); var data = NetworkPacketFactory.Message(MessageSerializer.Serialize(frame)); - return Send(0, false, data); + Send(0, false, data); } - public bool Response(byte[] data, int identity) + public void Response(byte[] data, int identity) { if (data == null) throw new ArgumentNullException(nameof(data)); if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Response] Socket status: {Status}"); - - return Send(0, false, NetworkPacketFactory.Response(data, identity)); + Send(0, false, NetworkPacketFactory.Response(data, identity)); } #endregion @@ -140,7 +154,7 @@ namespace ZeroLevel.Network try { if (type == FrameType.KeepAlive) return; - _incoming_queue.Add(new IncomingFrame + _incoming_queue.Enqueue(new IncomingFrame { data = data, type = type, @@ -223,75 +237,94 @@ namespace ZeroLevel.Network IncomingFrame frame = default(IncomingFrame); while (Status != SocketClientStatus.Disposed) { - try + if (_incoming_queue.TryDequeue(out frame)) { - if (_incoming_queue.TryTake(out frame, 100)) + try { - try + switch (frame.type) { - switch (frame.type) - { - case FrameType.Message: - Router?.HandleMessage(MessageSerializer.Deserialize(frame.data), this); - break; - case FrameType.Request: + case FrameType.Message: + Router?.HandleMessage(MessageSerializer.Deserialize(frame.data), this); + break; + case FrameType.Request: + { + Router?.HandleRequest(MessageSerializer.Deserialize(frame.data), this, frame.identity, (id, response) => { - Router?.HandleRequest(MessageSerializer.Deserialize(frame.data), this, frame.identity, (id, response) => + if (response != null) { - if (response != null) - { - this.Response(response, id); - } - }); - } - break; - case FrameType.Response: - { - _requests.Success(frame.identity, frame.data); - } - break; - } - } - catch (Exception ex) - { - Log.SystemError(ex, "[SocketClient.IncomingFramesJob] Handle frame"); + this.Response(response, id); + } + }); + } + break; + case FrameType.Response: + { + _requests.Success(frame.identity, frame.data); + } + break; } } - } - catch (Exception ex) - { - Log.SystemError(ex, "[SocketClient.IncomingFramesJob] _incoming_queue.Take"); - if (Status != SocketClientStatus.Disposed) + catch (Exception ex) { - _incoming_queue.Dispose(); - _incoming_queue = new BlockingCollection(); + Log.SystemError(ex, "[SocketClient.IncomingFramesJob] Handle frame"); } - continue; + } + else + { + Thread.Sleep(100); } } } - private bool Send(int id, bool is_request, byte[] data) + private void OutcomingFramesJob() { - if (Status == SocketClientStatus.Working) + while (Status != SocketClientStatus.Disposed) { - try + if (Status == SocketClientStatus.Working) { - if (is_request) - { - _requests.StartSend(id); + if (_outcomingFrameEvent.Wait(100)) + { + _outcomingFrameEvent.Reset(); } - var sended = _clientSocket.Send(data, data.Length, SocketFlags.None); - return sended == data.Length; + while (_outcoming_queue.TryDequeue(out var frame)) + { + try + { + if (frame.is_request) + { + _requests.StartSend(frame.identity); + } + _clientSocket.Send(frame.data, frame.data.Length, SocketFlags.None); + //var sended = _clientSocket.Send(frame.data, frame.data.Length, SocketFlags.None); + //return sended == frame.data.Length; + } + catch (Exception ex) + { + Log.SystemError(ex, $"[SocketClient.OutcomingFramesJob] _str_clientSocketeam.Send"); + Broken(); + OnDisconnect(this); + } + finally + { + _outcomingFramesPool.Return(frame); + } + } } - catch (Exception ex) + else { - Log.SystemError(ex, $"[SocketClient.SendFramesJob] _str_clientSocketeam.Send"); - Broken(); - OnDisconnect(this); + Thread.Sleep(400); } } - return false; + } + + private void Send(int id, bool is_request, byte[] data) + { + var frame = _outcomingFramesPool.Get(); + frame.data = data; + frame.identity = id; + frame.is_request = is_request; + _outcoming_queue.Enqueue(frame); + _outcomingFrameEvent.Set(); } #endregion diff --git a/ZeroLevel/Services/Semantic/LanguageDictionary.cs b/ZeroLevel/Services/Semantic/LanguageDictionary.cs index f44f5fd..bb7ab0d 100644 --- a/ZeroLevel/Services/Semantic/LanguageDictionary.cs +++ b/ZeroLevel/Services/Semantic/LanguageDictionary.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using ZeroLevel.Services.Semantic; using ZeroLevel.Services.Serialization; namespace ZeroLevel.Services.Semantic diff --git a/ZeroLevel/Services/Semantic/WordTokenizer.cs b/ZeroLevel/Services/Semantic/WordTokenizer.cs index 9b6c0b4..a2bb5e9 100644 --- a/ZeroLevel/Services/Semantic/WordTokenizer.cs +++ b/ZeroLevel/Services/Semantic/WordTokenizer.cs @@ -44,5 +44,41 @@ namespace ZeroLevel.Services.Semantic _pool.Return(buffer); } } + + public static IEnumerable TokenizeCaseSensitive(string text) + { + int index = 0; + bool first = true; + var buffer = _pool.Rent(ARRAY_SIZE); + try + { + for (int i = 0; i < text?.Length; i++) + { + if (first && Char.IsLetter(text[i])) + { + first = false; + buffer[index++] = text[i]; + } + else if (first == false && Char.IsLetterOrDigit(text[i])) + { + buffer[index++] = text[i]; + } + else if (index > 0) + { + yield return new string(buffer, 0, index); + index = 0; + first = true; + } + } + if (index > 0) + { + yield return new string(buffer, 0, index); + } + } + finally + { + _pool.Return(buffer); + } + } } } diff --git a/ZeroLevel/Services/Text/SuffixAutomata.cs b/ZeroLevel/Services/Text/SuffixAutomata.cs new file mode 100644 index 0000000..80fd5c8 --- /dev/null +++ b/ZeroLevel/Services/Text/SuffixAutomata.cs @@ -0,0 +1,159 @@ +using System.Collections.Generic; +using System.Linq; + +namespace ZeroLevel.Services.Text +{ + internal struct state + { + public int len; + public int link; + public Dictionary next; + } + + public class SuffixAutomata + { + const int MAXLEN = 100000; + private state[] st = new state[MAXLEN * 2]; + int sz, last; + + public void Init(bool is_reused = false) + { + sz = last = 0; + st[0].len = 0; + st[0].link = -1; + ++sz; + // этот код нужен, только если автомат строится много раз для разных строк: + for (int i = 0; i < MAXLEN * 2; ++i) + { + st[i].next = new Dictionary(); + } + } + + public void Extend(char c) + { + int cur = sz++; + st[cur].len = st[last].len + 1; + int p; + for (p = last; p != -1 && !st[p].next.ContainsKey(c); p = st[p].link) + st[p].next[c] = cur; + if (p == -1) + st[cur].link = 0; + else + { + int q = st[p].next[c]; + if (st[p].len + 1 == st[q].len) + st[cur].link = q; + else + { + int clone = sz++; + st[clone].len = st[p].len + 1; + st[clone].next = st[q].next; + st[clone].link = st[q].link; + for (; p != -1 && st[p].next.ContainsKey(c) && st[p].next[c] == q; p = st[p].link) + st[p].next[c] = clone; + st[q].link = st[cur].link = clone; + } + } + last = cur; + } + + public bool IsSubstring(string w) + { + if (string.IsNullOrEmpty(w)) return true; + bool fail = false; + int n, si = 0; + for (; si < last; si++) + { + if (st[si].next.ContainsKey(w[0])) + { + var start = st[si]; + for (int i = 0; i < w.Length; i++) + { + if (start.next.ContainsKey(w[i]) == false) + { + fail = true; + break; + } + n = start.next[w[i]]; + start = st[n]; + } + break; + } + } + if (si == last) + { + fail = true; + } + return (!fail); + } + + public string Intersection(string t) + { + var entries = st.Where(x => x.next.ContainsKey(t[0])).ToArray(); + var candidates = entries + .Select(s => Intersection(s, t)); + if (candidates != null && candidates.Any()) + { + var max = candidates.Max(s => s?.Length ?? 0); + return candidates.FirstOrDefault(c => c != null && c.Length == max); + } + return null; + /* + int v = 0, l = 0, best = 0, bestpos = 0; + for (int i = 0; i < (int)t.Length; ++i) + { + while (v > 0 && !st[v].next.ContainsKey(t[i])) + { + v = st[v].link; + l = st[v].len; + } + if (st[v].next.ContainsKey(t[i])) + { + v = st[v].next[t[i]]; + ++l; + } + if (l > best) + { + best = l; + bestpos = i; + } + } + var start = bestpos - best + 1; + var length = best; + if (start >= 0 && start < t.Length && (start + length) <= t.Length) + return t.Substring(start, length); + return null; + */ + } + + private string Intersection(state entry, string t) + { + int v = 0, l = 0, best = 0, bestpos = 0; + for (int i = 0; i < (int)t.Length; ++i) + { + while (v > 0 && !entry.next.ContainsKey(t[i])) + { + v = entry.link; + l = entry.len; + entry = st[v]; + } + if (entry.next.ContainsKey(t[i])) + { + v = entry.next[t[i]]; + entry = st[v]; + ++l; + } + if (l > best) + { + best = l; + bestpos = i; + } + } + var start = bestpos - best + 1; + var length = best; + if (start >= 0 && start < t.Length && (start + length) <= t.Length) + return t.Substring(start, length); + return null; + } + } +} diff --git a/ZeroLevel/Services/Text/TStringReader.cs b/ZeroLevel/Services/Text/TStringReader.cs index 4f5cce4..4875c02 100644 --- a/ZeroLevel/Services/Text/TStringReader.cs +++ b/ZeroLevel/Services/Text/TStringReader.cs @@ -39,9 +39,13 @@ public void SkipBreaks() { - while (EOF == false && char.IsWhiteSpace(Current)) Move(); + while (EOF == false && (Current == '\n' || Current == '\r')) Move(); } + public bool IsBrackets() => EOF == false && (Current == '\'' || Current == '"' || Current == '”'); + public bool IsBreak() => EOF == false && (Current == '\n' || Current == '\r'); + public bool IsWhiteSpace() => EOF == false && char.IsWhiteSpace(Current); + public bool MoveBack() { _position = _position - 1; @@ -84,7 +88,7 @@ index++; identity = _template.Substring(offset, index - offset); } - return identity.ToLowerInvariant(); + return identity; } public string ReadWord() diff --git a/ZeroLevel/Services/Transliteration.cs b/ZeroLevel/Services/Transliteration.cs index f24c5a9..1a15def 100644 --- a/ZeroLevel/Services/Transliteration.cs +++ b/ZeroLevel/Services/Transliteration.cs @@ -22,7 +22,7 @@ namespace ZeroLevel.Services output = Regex.Replace(output, @"\s|\.|\(", " "); output = Regex.Replace(output, @"\s+", " "); - output = Regex.Replace(output, @"[^\s\w\d-]", ""); + //output = Regex.Replace(output, @"[^\s\w\d-]", ""); output = output.Trim(); Dictionary tdict = GetDictionaryByType(type); @@ -145,7 +145,7 @@ namespace ZeroLevel.Services gost.Add("«", ""); gost.Add("»", ""); gost.Add("—", "-"); - gost.Add(" ", "-"); + gost.Add(" ", " "); iso.Add("Є", "YE"); iso.Add("І", "I"); @@ -223,7 +223,7 @@ namespace ZeroLevel.Services iso.Add("«", ""); iso.Add("»", ""); iso.Add("—", "-"); - iso.Add(" ", "-"); + iso.Add(" ", " "); } } } diff --git a/ZeroLevel/Services/Web/HtmlUtility.cs b/ZeroLevel/Services/Web/HtmlUtility.cs index ab1b148..5c5318f 100644 --- a/ZeroLevel/Services/Web/HtmlUtility.cs +++ b/ZeroLevel/Services/Web/HtmlUtility.cs @@ -346,7 +346,7 @@ namespace ZeroLevel.Services.Web if (cursor.Current == '&') { cursor.Move(); - var identity = cursor.ReadIdentity(); + var identity = cursor.ReadIdentity().ToLowerInvariant(); cursor.MoveBack(); if (_entityMap.ContainsKey(identity + ";")) { diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index 8ca8510..9122ef0 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -6,7 +6,7 @@ ogoun ogoun - 3.3.6.3 + 3.3.6.4 Configuration floating number convert fix https://github.com/ogoun/Zero/wiki Copyright Ogoun 2022 @@ -14,8 +14,8 @@ https://github.com/ogoun/Zero git - 3.3.6.3 - 3.3.6.3 + 3.3.6.4 + 3.3.6.4 AnyCPU;x64;x86 zero.png full