Improve quality

.gitignore

@@ -30,6 +30,7 @@ bower_components
 *.nupkg
 *.p7s
 *.sln.ide
+*.onnx
 [Bb]in
 [Dd]ebug*/
 [Oo]bj/

AutoLoader/AppContainer/AppContainer.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

AutoLoader/AppContainer/Class1.cs

@@ -0,0 +1,50 @@
+namespace AppContainer
+{
+
+    public class Endpoint
+    {
+        public string VersionEndpoint { get; set; }
+        public string AppEndpoint { get; set; }
+        public string PingEndpoint { get; set; }
+        public string Token { get; set; }
+    }
+    public class HostSettings
+    {
+        public IEnumerable<Endpoint> Endpoints { get; set; }
+        public string EntryClass { get; set; }
+        public string EntryMethod { get; set; }
+    }
+
+    public class Host
+    {
+        private readonly HostSettings _settings;
+        public Host(HostSettings settings)
+        {
+            if (settings == null) throw new ArgumentNullException(nameof(settings));
+            _settings = settings;
+        }
+
+        public async Task<bool> HasUpdate()
+        {
+            if (_settings.Endpoints != null)
+            {
+                foreach (var ep in _settings.Endpoints)
+                {
+                    try
+                    {
+                        using (var client = new HttpClient())
+                        {
+                            client.DefaultRequestHeaders.Add("token", ep.Token);
+                            var response = await client.GetAsync(ep.PingEndpoint);
+                            response.EnsureSuccessStatusCode();
+                            
+                        }
+                    }
+                    catch (Exception ex)
+                    { 
+                    }
+                }
+            }
+        }
+    }
+}

ConnectionTest/Client/Client.csproj

@@ -1,9 +1,10 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>net6.0</TargetFramework>
-    <Platforms>AnyCPU;x64;x86</Platforms>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>

ConnectionTest/Client/Program.cs

@@ -1,7 +1,4 @@
-using System;
+using System.Net;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net;
 using ZeroLevel;
 using ZeroLevel.Network;
 using ZeroLevel.Services.HashFunctions;
@@ -60,6 +57,9 @@ namespace Client
     {
         private readonly static XXHashUnsafe _hash = new XXHashUnsafe(667);
 
+        private static long _successSend = 0;
+        private static long _faultSend = 0;
+
         static void Main(string[] args)
         {
             Log.AddConsoleLogger();
@@ -68,6 +68,12 @@ namespace Client
             var port = ReadPort();
             ex.RoutesStorage.Set("server", new IPEndPoint(address, port));
 
+            Sheduller.RemindEvery(TimeSpan.FromMilliseconds(300), () => 
+            {
+                Console.SetCursorPosition(0, 0);
+                Console.WriteLine($"Success/Fault network: {_successSend} / {_faultSend}");
+            });
+
             uint index = 0;
             while (true)
             {
@@ -99,7 +105,7 @@ namespace Client
             var info = new Info { Checksum = full_checksum, Id = id, Length = length };
             if (client.Request<Info, bool>("start", info, res =>
             {
-                Log.Info($"Success start sending packet '{id}'");
+                Interlocked.Increment(ref _successSend);
             }))
             {
                 uint size = 1;
@@ -111,11 +117,15 @@ namespace Client
                     {
                         if (!res)
                         {
-                            Log.Info($"Fault server incoming packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes.");
+                            Interlocked.Increment(ref _faultSend);
+                        }
+                        else
+                        {
+                            Interlocked.Increment(ref _successSend);
                         }
                     }))
                     {
-                        Log.Warning($"Can't start send packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes. No connection");
+                        Interlocked.Increment(ref _faultSend);
                     }
                     offset += size;
                     size += 1;
@@ -124,7 +134,7 @@ namespace Client
             }
             else
             {
-                Log.Warning($"Can't start send packet '{id}'. No connection");
+                Interlocked.Increment(ref _faultSend);
             }
         }
 
@@ -135,7 +145,7 @@ namespace Client
             var info = new Info { Checksum = full_checksum, Id = id, Length = length };
             if (exchange.Request<Info, bool>("server", "start", info))
             {
-                Log.Info($"Success start sending packet '{id}'");
+                Interlocked.Increment(ref _successSend);
                 uint size = 4096;
                 uint offset = 0;
                 while (offset < payload.Length)
@@ -143,7 +153,11 @@ namespace Client
                     var fragment = GetFragment(id, payload, offset, size);
                     if (!exchange.Request<Fragment, bool>("server", "part", fragment))
                     {
-                            Log.Info($"Fault server incoming packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes.");
+                        Interlocked.Increment(ref _faultSend);
+                    }
+                    else
+                    {
+                        Interlocked.Increment(ref _successSend);
                     }
                     offset += size;
                 }
@@ -151,7 +165,7 @@ namespace Client
             }
             else
             {
-                Log.Warning($"Can't start send packet '{id}'. No connection");
+                Interlocked.Increment(ref _faultSend);
             }
         }
 

ConnectionTest/Client/Properties/PublishProfiles/FolderProfile.pubxml

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
-https://go.microsoft.com/fwlink/?LinkID=208121. 
--->
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <PropertyGroup>
-    <PublishProtocol>FileSystem</PublishProtocol>
-    <Configuration>Release</Configuration>
-    <Platform>Any CPU</Platform>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
-    <PublishDir>bin\Release\netcoreapp3.1\publish\</PublishDir>
-  </PropertyGroup>
-</Project>

ConnectionTest/Server/Program.cs

@@ -6,6 +6,7 @@ using ZeroLevel.Services.Serialization;
 
 namespace Server
 {
+    //netsh advfirewall firewall add rule name="ClientTest" dir=in action=allow protocol=TCP localport=5016
     public class Info
         : IBinarySerializable
     {

TestApp/HybrydLock.cs

@@ -1,28 +0,0 @@
-using System.Threading;
-
-namespace TestApp
-{
-    public class HybrydLock
-    {
-        private AutoResetEvent _lock = new AutoResetEvent(false);
-        private volatile int _counter = 0;
-
-        public void Enter()
-        {
-            if (Interlocked.Increment(ref _counter) == 1)
-            {
-                return;
-            }
-            _lock.WaitOne();
-        }
-
-        public void Leave()
-        {
-            if (Interlocked.Decrement(ref _counter) == 0)
-            {
-                return;
-            }
-            _lock.Set();
-        }
-    }
-}

TestApp/MyService.cs

@@ -1,72 +0,0 @@
-using System;
-using System.Net;
-using System.Threading;
-using ZeroLevel;
-using ZeroLevel.Network;
-using ZeroLevel.Services.Applications;
-
-namespace TestApp
-{
-    public class MyService
-        : BaseZeroService
-    {
-        public MyService()
-            : base()
-        {
-        }
-
-        protected override void StartAction()
-        {
-            Log.Info("Started");
-            ReadServiceInfo();
-            var host = UseHost(8800);
-            AutoregisterInboxes(host);
-            host.OnConnect += Host_OnConnect;
-            host.OnDisconnect += Host_OnDisconnect;
-
-            int counter = 0;
-            Sheduller.RemindEvery(TimeSpan.FromSeconds(1), () =>
-            {
-                Log.Info($"RPS: {counter}");
-                Interlocked.Exchange(ref counter, 0);
-            });
-
-            Exchange.RoutesStorage.Set("test.app", new IPEndPoint(IPAddress.Loopback, 8800));
-
-            while (true)
-            {
-                try
-                {
-                    //var s = Exchange.Request<int>("test.app", "counter");
-                    Interlocked.Add(ref counter, Exchange.Request<int>("test.app", "counter"));
-                }
-                catch(Exception ex)
-                {
-                    Log.Error(ex, "Request fault");
-                    Thread.Sleep(300);
-                }
-            }
-        }
-
-        private void Host_OnDisconnect(ISocketClient obj)
-        {
-            Log.Info($"Client '{obj.Endpoint.Address}:{obj.Endpoint.Port}' disconnected");
-        }
-
-        private void Host_OnConnect(IClient obj)
-        {
-            Log.Info($"Client '{obj.Socket.Endpoint.Address}:{obj.Socket.Endpoint.Port}' connected");
-        }
-
-        [ExchangeReplierWithoutArg("counter")]
-        public int GetCounter(ISocketClient client)
-        {
-            return 1;
-        }
-
-        protected override void StopAction()
-        {
-            Log.Info("Stopped");
-        }
-    }
-}

TestApp/PersonDetector.cs

@@ -0,0 +1,44 @@
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.PixelFormats;
+using System.Collections.Generic;
+using System.Linq;
+using ZeroLevel.NN.Architectures.YoloV5;
+using ZeroLevel.NN.Models;
+using ZeroLevel.NN.Services;
+
+namespace TestApp
+{
+    public class PersonDetector
+    {
+        private const string MODEL_PATH = @"nnmodels/Yolo5S/yolov5s327e.onnx";
+        private readonly Yolov5Detector _detector;
+        private float _threshold = 0.17f;
+        public PersonDetector()
+        {
+            _detector = new Yolov5Detector(MODEL_PATH, gpu: false);
+        }
+
+        public IEnumerable<YoloPrediction> Detect(string imagePath)
+        {
+            using (Image<Rgb24> image = Image.Load<Rgb24>(imagePath))
+            {
+                var t_predictions = _detector.PredictMultiply(image, true, _threshold);
+                t_predictions.Apply(p =>
+                {
+                    p.Cx /= image.Width;
+                    p.Cy /= image.Height;
+                });
+                if (t_predictions != null)
+                {
+                    t_predictions.RemoveAll(p => (p.W * image.Width) < 10.0f || (p.H * image.Height) < 10.0f);
+                }
+                if (t_predictions.Count > 0)
+                {
+                    NMS.Apply(t_predictions);
+                    return t_predictions;
+                }
+            }
+            return Enumerable.Empty<YoloPrediction>();
+        }
+    }
+}

TestApp/Program.cs

@@ -1,84 +1,11 @@
-using Newtonsoft.Json;
+namespace TestApp
-using System;
-using System.Net;
-using System.Threading;
-using ZeroLevel;
-using ZeroLevel.Logging;
-using ZeroLevel.Network;
-using ZeroLevel.Services.Serialization;
-using ZeroLevel.Services.Trees;
-
-namespace TestApp
 {
-    public class TestQuery
-    {
-        public string Name { get; set; }
-        public int Age { get; set; }
-        public string[] Roles { get; set; }
-    }
-
     internal static class Program
     {
-        private static string Serialize(object instance)
-        {
-            return JsonConvert.SerializeObject(instance);
-        }
-
         private static void Main(string[] args)
         {
-            Configuration.Save(Configuration.ReadFromApplicationConfig());
+            var detector = new PersonDetector();
-            Bootstrap.Startup<MyService>(args,
+            var predictions = detector.Detect(@"E:\Desktop\test\1.JPG");
-                () => Configuration.ReadSetFromIniFile("config.ini"))
-                .EnableConsoleLog(LogLevel.System | LogLevel.FullDebug)
-                //.UseDiscovery()
-                .Run()
-                .WaitWhileStatus(ZeroServiceStatus.Running)
-                .Stop();
-            Bootstrap.Shutdown();
-        }
-
-        static void SimpleCSTest()
-        {
-            var server_router = new Router();
-            server_router.RegisterInbox<string>("test", (c, line) =>
-            {
-                Console.WriteLine(line);
-            });
-
-            server_router.RegisterInbox<string, string>("req", (c, line) =>
-            {
-                Console.WriteLine($"Request: {line}");
-                return line.ToUpperInvariant();
-            });
-
-            var server = new SocketServer(new System.Net.IPEndPoint(IPAddress.Any, 666), server_router);
-
-
-            var client_router = new Router();
-            var client = new SocketClient(new IPEndPoint(IPAddress.Loopback, 666), client_router);
-
-            var frm = FrameFactory.Create("req", MessageSerializer.SerializeCompatible("Hello world"));
-
-            while (Console.KeyAvailable == false)
-            {
-                client.Request(frm, data =>
-                {
-                    var line = MessageSerializer.DeserializeCompatible<string>(data);
-                    Console.WriteLine($"Response: {line}");
-                });
-                Thread.Sleep(2000);
-            }
-        }
-
-        public static double[] Generate(int vector_size)
-        {
-            var rnd = new Random((int)Environment.TickCount);
-            var vector = new double[vector_size];
-            for (int i = 0; i < vector_size; i++)
-            {
-                vector[i] = 50.0d - rnd.NextDouble() * 100.0d;
-            }
-            return vector;
         }
     }
 }

TestApp/TestApp.csproj

@@ -11,6 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\ZeroLevel.NN\ZeroLevel.NN.csproj" />
     <ProjectReference Include="..\ZeroLevel\ZeroLevel.csproj" />
   </ItemGroup>
 
@@ -18,6 +19,9 @@
     <None Update="config.ini">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="nnmodels\Yolo5S\yolov5s327e.onnx">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
 </Project>

ZeroLevel.NN/Architectures/AgeDetectors/GenderAgeEstimator.cs

@@ -0,0 +1,37 @@
+using Microsoft.ML.OnnxRuntime.Tensors;
+using SixLabors.ImageSharp;
+using ZeroLevel.NN.Models;
+
+namespace ZeroLevel.NN
+{
+    public class GenderAgeEstimator
+        : SSDNN
+    {
+        private const int INPUT_WIDTH = 64;
+        private const int INPUT_HEIGHT = 64;
+
+        public GenderAgeEstimator(string modelPath, bool gpu = false) : base(modelPath, gpu)
+        {
+        }
+
+        public (Gender, int) Predict(Image image)
+        {
+            var input = MakeInput(image,
+                new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst)
+                .ApplyNormilization()
+                .ApplyAxeInversion());
+            return Predict(input);
+        }
+
+        public (Gender, int) Predict(Tensor<float> input)
+        {
+            float[] variances = null;
+            Extract(new Dictionary<string, Tensor<float>> { { "input", input } }, d =>
+            {
+                variances = d.First().Value.ToArray();
+            });
+            var gender = Argmax(variances[0..2]) == 0 ? Gender.Male : Gender.Feemale;
+            return (gender, (int)variances[2]);
+        }
+    }
+}

ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeEstimator.cs

@@ -19,7 +19,7 @@ namespace ZeroLevel.NN
     /// <summary>
     /// Input tensor is 1 x 3 x height x width with mean values 104, 117, 123. Input image have to be previously resized to 224 x 224 pixels and converted to BGR format. 
     /// </summary>
-    public class GoogleAgeDetector
+    public class GoogleAgeEstimator
         : SSDNN
     {
         private const int INPUT_WIDTH = 224;
@@ -28,7 +28,7 @@ namespace ZeroLevel.NN
 
         private Age[] _ageList = new[] { Age.From0To2, Age.From4To6, Age.From8To12, Age.From15To20, Age.From25To32, Age.From38To43, Age.From48To53, Age.From60To100 };
 
-        public GoogleAgeDetector(string modelPath, bool gpu = false) : base(modelPath, gpu)
+        public GoogleAgeEstimator(string modelPath, bool gpu = false) : base(modelPath, gpu)
         {
         }
 
@@ -48,7 +48,7 @@ namespace ZeroLevel.NN
             {
                 variances = d.First().Value.ToArray();
             });
-            var (number, index) = variances.Select((n, i) => (n, i)).Max();
+            var index = Argmax(variances);
             return _ageList[index];
         }
     }

ZeroLevel.NN/Architectures/FaceSeacrhService.cs

@@ -1,6 +1,7 @@
 using SixLabors.ImageSharp;
 using SixLabors.ImageSharp.PixelFormats;
 using SixLabors.ImageSharp.Processing;
+using ZeroLevel;
 using ZeroLevel.NN;
 using ZeroLevel.NN.Models;
 
@@ -127,6 +128,8 @@ namespace Zero.NN.Services
                     var around_x2 = centerFaceX + radius;
                     var around_y1 = centerFaceY - radius;
                     var around_y2 = centerFaceY + radius;
+                    try
+                    {
                         using (var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2))
                         {
                             var matrix = Face.GetTransformMatrix(face);
@@ -148,6 +151,15 @@ namespace Zero.NN.Services
                             }*/
                         }
                     }
+                    catch (Exception ex)
+                    {
+                        Log.SystemError(ex, "[FaceSeacrhService.GetEmbeddings]");
+                        using (var faceImage = ImagePreprocessor.Crop(image, face.X1, face.Y1, face.X2, face.Y2))
+                        {
+                            vector = _encoder.Predict(faceImage);
+                        }
+                    }
+                }
                 else
                 {
                     using (var faceImage = ImagePreprocessor.Crop(image, face.X1, face.Y1, face.X2, face.Y2))
@@ -162,5 +174,53 @@ namespace Zero.NN.Services
                 };
             }
         }
+
+
+        public IEnumerable<(FaceEmbedding, Image)> GetEmbeddingsAndCrop(Image<Rgb24> image)
+        {
+            int width = image.Width;
+            int heigth = image.Height;
+            var faces = _detector.Predict(image);
+            foreach (var face in faces)
+            {
+                Face.FixInScreen(face, width, heigth);
+                float[] vector;
+                if (_useFaceAlign)
+                {
+                    int x = (int)face.X1;
+                    int y = (int)face.Y1;
+                    int w = (int)(face.X2 - face.X1);
+                    int h = (int)(face.Y2 - face.Y1);
+                    var radius = (float)Math.Sqrt(w * w + h * h) / 2f;
+                    var centerFaceX = (face.X2 + face.X1) / 2.0f;
+                    var centerFaceY = (face.Y2 + face.Y1) / 2.0f;
+                    var around_x1 = centerFaceX - radius;
+                    var around_x2 = centerFaceX + radius;
+                    var around_y1 = centerFaceY - radius;
+                    var around_y2 = centerFaceY + radius;
+                    var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2);
+                    var matrix = Face.GetTransformMatrix(face);
+                    var builder = new AffineTransformBuilder();
+                    builder.AppendMatrix(matrix);
+                    faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic));
+                    vector = _encoder.Predict(faceImage);
+                    yield return (new FaceEmbedding
+                    {
+                        Face = face,
+                        Vector = vector
+                    }, faceImage);
+                }
+                else
+                {
+                    var faceImage = ImagePreprocessor.Crop(image, face.X1, face.Y1, face.X2, face.Y2);
+                    vector = _encoder.Predict(faceImage);
+                    yield return (new FaceEmbedding
+                    {
+                        Face = face,
+                        Vector = vector
+                    }, faceImage);
+                }
+            }
+        }
     }
 }

ZeroLevel.NN/Architectures/YoloV5/Yolov5Detector.cs

@@ -0,0 +1,144 @@
+using Microsoft.ML.OnnxRuntime.Tensors;
+using SixLabors.ImageSharp;
+using ZeroLevel.NN.Models;
+
+namespace ZeroLevel.NN.Architectures.YoloV5
+{
+    public class Yolov5Detector
+        : SSDNN
+    {
+        private int INPUT_WIDTH = 640;
+        private int INPUT_HEIGHT = 640;
+        private int CROP_WIDTH = 1440;
+        private int CROP_HEIGHT = 1440;
+
+        public Yolov5Detector(string modelPath, int inputWidth = 640, int inputHeight = 640, bool gpu = false)
+            : base(modelPath, gpu)
+        {
+            INPUT_HEIGHT = inputHeight;
+            INPUT_WIDTH = inputWidth;
+        }
+
+        public List<YoloPrediction> Predict(Image image, float threshold)
+        {
+            var input = MakeInput(image,
+                new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst)
+                .ApplyNormilization()
+                .ApplyAxeInversion());
+            return Predict(input, threshold);
+        }
+
+        public List<YoloPrediction> PredictMultiply(Image image, bool withFullResizedImage, float threshold)
+        {
+            var input = MakeInputBatch(image,
+                new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst)
+                .ApplyNormilization()
+                .ApplyAxeInversion()
+                .UseCrop(CROP_WIDTH, CROP_HEIGHT, withFullResizedImage, true));
+            return PredictMultiply(input, threshold);
+        }
+
+        public List<YoloPrediction> Predict(Tensor<float> input, float threshold)
+        {
+            var result = new List<YoloPrediction>();
+            Extract(new Dictionary<string, Tensor<float>> { { "images", input } }, d =>
+            {
+                var output = d["output"];
+                /*
+                var output350 = d["350"];
+                var output498 = d["498"];
+                var output646 = d["646"];
+                */
+                if (output != null && output != null)
+                {
+                    var relative_koef_x = 1.0f / INPUT_WIDTH;
+                    var relative_koef_y = 1.0f / INPUT_HEIGHT;
+                    for (int box = 0; box < output.Dimensions[1]; box++)
+                    {
+                        var conf = output[0, box, 4]; // уверенность в наличии любого объекта
+                        if (conf > threshold)
+                        {
+                            var class_confidense = output[0, box, 5]; // уверенность в наличии объекта класса person
+                            if (class_confidense > threshold)
+                            {
+                                // Перевод относительно входа модели в относительные координаты
+                                var cx = output[0, box, 0] * relative_koef_x;
+                                var cy = output[0, box, 1] * relative_koef_y;
+                                var h = output[0, box, 2] * relative_koef_y;
+                                var w = output[0, box, 3] * relative_koef_x;
+                                result.Add(new YoloPrediction
+                                {
+                                    Cx = cx,
+                                    Cy = cy,
+                                    W = w,
+                                    H = h,
+                                    Class = 0,
+                                    Label = "0",
+                                    Score = conf
+                                });
+                            }
+                        }
+                    }
+                }
+            });
+            return result;
+        }
+
+        public List<YoloPrediction> PredictMultiply(ImagePredictionInput[] inputs, float threshold)
+        {
+            var result = new List<YoloPrediction>();
+            var relative_koef_x = 1.0f / (float)INPUT_WIDTH;
+            var relative_koef_y = 1.0f / (float)INPUT_HEIGHT;
+            foreach (var input in inputs)
+            {
+                Extract(new Dictionary<string, Tensor<float>> { { "images", input.Tensor } }, d =>
+                {
+                    var output = d["output"];
+                    /*
+                    var output350 = d["350"];
+                    var output498 = d["498"];
+                    var output646 = d["646"];
+                    */
+                    if (output != null && output != null)
+                    {
+                        for (int index = 0; index < input.Count; index++)
+                        {
+                            var real_koef_x = (float)input.Offsets[index].Width / (float)INPUT_WIDTH;
+                            var real_koef_y = (float)input.Offsets[index].Height / (float)INPUT_HEIGHT;
+                            for (int box = 0; box < output.Dimensions[1]; box++)
+                            {
+                                var conf = output[index, box, 4]; // уверенность в наличии любого объекта
+                                if (conf > threshold)
+                                {
+                                    var class_confidense = output[index, box, 5]; // уверенность в наличии объекта класса person
+                                    if (class_confidense > threshold)
+                                    {
+                                        // Перевод относительно входа модели в относительные координаты
+                                        var cx = output[index, box, 0] * real_koef_x;
+                                        var cy = output[index, box, 1] * real_koef_y;
+                                        var h = output[index, box, 2] * relative_koef_y;
+                                        var w = output[index, box, 3] * relative_koef_x;
+                                        // Перевод в координаты отнисительно текущего смещения
+                                        cx += input.Offsets[index].X;
+                                        cy += input.Offsets[index].Y;
+                                        result.Add(new YoloPrediction
+                                        {
+                                            Cx = cx,
+                                            Cy = cy,
+                                            W = w,
+                                            H = h,
+                                            Class = 0,
+                                            Label = "0",
+                                            Score = conf
+                                        });
+                                    }
+                                }
+                            }
+                        }
+                    }
+                });
+            }
+            return result;
+        }
+    }
+}

ZeroLevel.NN/Models/Gender.cs

@@ -0,0 +1,9 @@
+namespace ZeroLevel.NN.Models
+{
+    public enum Gender
+    {
+        Unknown = 0, 
+        Male = 1, 
+        Feemale = 2
+    }
+}

ZeroLevel.NN/Models/ImagePredictionInput.cs

@@ -2,7 +2,7 @@
 
 namespace ZeroLevel.NN.Models
 {
-    public class PredictionInput
+    public class ImagePredictionInput
     {
         public Tensor<float> Tensor;
         public OffsetBox[] Offsets;

ZeroLevel.NN/Models/ImagePreprocessorOptions.cs

@@ -2,7 +2,7 @@
 {
     public class ImagePreprocessorOptions
     {
-        private const float PIXEL_NORMALIZATION_SCALE = 1f / 255f;
+        private const float PIXEL_NORMALIZATION_SCALE = 1.0f / 255.0f;
         public ImagePreprocessorOptions(int inputWidth, int inputHeight, PredictorChannelType channelType)
         {
             this.InputWidth = inputWidth;

ZeroLevel.NN/Models/YoloPrediction.cs

@@ -0,0 +1,66 @@
+using ZeroLevel.Services.Serialization;
+
+namespace ZeroLevel.NN.Models
+{
+    public class YoloPrediction
+        : IBinarySerializable
+    {
+        public int Class { get; set; }
+        public float Cx { get; set; }
+        public float Cy { get; set; }
+        public float W { get; set; }
+        public float H { get; set; }
+        public float Score { get; set; }
+        public string Label { get; set; }
+
+        public float X { get { return Cx - W / 2.0f; } }
+        public float Y { get { return Cy - W / 2.0f; } }
+
+        public float Area { get { return W * H; } }
+
+        public string Description
+        {
+            get
+            {
+                return $"{Label} ({(int)(Score * 100)} %)";
+            }
+        }
+
+        public float this[int index]
+        {
+            get
+            {
+                switch (index)
+                {
+                    case 0: return Cx;
+                    case 1: return Cy;
+                    case 2: return Cx + W;
+                    case 3: return Cy + H;
+                }
+                return 0;
+            }
+        }
+
+        public void Deserialize(IBinaryReader reader)
+        {
+            this.Cx = reader.ReadFloat();
+            this.Cy = reader.ReadFloat();
+            this.W = reader.ReadFloat();
+            this.H = reader.ReadFloat();
+            this.Class = reader.ReadInt32();
+            this.Score = reader.ReadFloat();
+            this.Label = reader.ReadString();
+        }
+
+        public void Serialize(IBinaryWriter writer)
+        {
+            writer.WriteFloat(this.Cx);
+            writer.WriteFloat(this.Cy);
+            writer.WriteFloat(this.W);
+            writer.WriteFloat(this.H);
+            writer.WriteInt32(this.Class);
+            writer.WriteFloat(this.Score);
+            writer.WriteString(this.Label);
+        }
+    }
+}

ZeroLevel.NN/Services/ImagePreprocessor.cs

@@ -8,6 +8,8 @@ namespace ZeroLevel.NN
 {
     public static class ImagePreprocessor
     {
+        private static Action<Tensor<float>, float, int, int, int, int> _precompiledChannelFirstAction = new Action<Tensor<float>, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, c, i, j] = v; });
+        private static Action<Tensor<float>, float, int, int, int, int> _precompiledChannelLastAction = new Action<Tensor<float>, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, i, j, c] = v; });
         private static Func<byte, int, float> PixelToTensorMethod(ImagePreprocessorOptions options)
         {
             if (options.Normalize)
@@ -42,26 +44,51 @@ namespace ZeroLevel.NN
             return new Func<byte, int, float>((b, _) => (float)b);
         }
 
+        //private static int CalculateFragmentsCount(Image image, ImagePreprocessorOptions options)
+        //{
+        //    int count = (options.Crop.SaveOriginal ? 1 : 0);
+        //    var Sw = image.Width;           // ширина оригинала
+        //    var Sh = image.Height;          // высота оригинала
+
+        //    var CRw = options.Crop.Width;   // ширина кропа
+        //    var CRh = options.Crop.Height;  // высота кропа
+
+        //    var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw;   // сдвиг по OX к следующему кропу
+        //    var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh;  // сдвиг по OY к следующему кропу
+
+        //    for (int x = 0; x < Sw; x += Dx)
+        //    {
+        //        for (int y = 0; y < Sh; y += Dy)
+        //        {
+        //            count++;
+        //        }
+        //    }
+        //    return count;
+        //}
         private static int CalculateFragmentsCount(Image image, ImagePreprocessorOptions options)
         {
-            int count = 0;
+            int count = (options.Crop.SaveOriginal ? 1 : 0);
-            var xs = options.Crop.Overlap ? (int)(options.Crop.Width * options.Crop.OverlapKoefWidth) : options.Crop.Width;
+            var Sw = image.Width;           // ширина оригинала
-            var ys = options.Crop.Overlap ? (int)(options.Crop.Height * options.Crop.OverlapKoefHeight) : options.Crop.Height;
+            var Sh = image.Height;          // высота оригинала
-            for (var x = 0; x < image.Width - xs; x += xs)
+
+            var CRw = options.InputWidth;   // ширина кропа (равна ширине входа, т.к. изображение отресайзено подобающим образом)
+            var CRh = options.InputHeight;  // высота кропа (равна высоте входа, т.к. изображение отресайзено подобающим образом)
+
+            var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw;   // сдвиг по OX к следующему кропу
+            var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh;  // сдвиг по OY к следующему кропу
+
+            for (int x = 0; x < Sw; x += Dx)
             {
-                for (var y = 0; y < image.Height - ys; y += ys)
+                for (int y = 0; y < Sh; y += Dy)
                 {
                     count++;
                 }
             }
             return count;
         }
-
         private static void FillTensor(Tensor<float> tensor, Image image, int index, ImagePreprocessorOptions options, Func<byte, int, float> pixToTensor)
         {
-            var append = options.ChannelType == PredictorChannelType.ChannelFirst
+            var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction;
-                ? new Action<Tensor<float>, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, c, i, j] = v; })
-                : new Action<Tensor<float>, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, i, j, c] = v; });
 
             ((Image<Rgb24>)image).ProcessPixelRows(pixels =>
             {
@@ -112,6 +139,59 @@ namespace ZeroLevel.NN
             });
         }
 
+        private static void FillTensor(Tensor<float> tensor, Image image, int startX, int startY, int w, int h, int index, ImagePreprocessorOptions options, Func<byte, int, float> pixToTensor)
+        {
+            var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction;
+
+            ((Image<Rgb24>)image).ProcessPixelRows(pixels =>
+            {
+                if (options.InvertXY)
+                {
+                    for (int y = startY; y < h; y++)
+                    {
+                        Span<Rgb24> pixelSpan = pixels.GetRowSpan(y);
+                        for (int x = startX; x < w; x++)
+                        {
+                            if (options.BGR)
+                            {
+                                append(tensor, pixToTensor(pixelSpan[x].B, 0), index, 0, y, x);
+                                append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, y, x);
+                                append(tensor, pixToTensor(pixelSpan[x].R, 2), index, 2, y, x);
+                            }
+                            else
+                            {
+                                append(tensor, pixToTensor(pixelSpan[x].R, 0), index, 0, y, x);
+                                append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, y, x);
+                                append(tensor, pixToTensor(pixelSpan[x].B, 2), index, 2, y, x);
+                            }
+                        }
+                    }
+                }
+                else
+                {
+                    for (int y = startY; y < h; y++)
+                    {
+                        Span<Rgb24> pixelSpan = pixels.GetRowSpan(y);
+                        for (int x = startX; x < w; x++)
+                        {
+                            if (options.BGR)
+                            {
+                                append(tensor, pixToTensor(pixelSpan[x].B, 0), index, 0, x, y);
+                                append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, x, y);
+                                append(tensor, pixToTensor(pixelSpan[x].R, 2), index, 2, x, y);
+                            }
+                            else
+                            {
+                                append(tensor, pixToTensor(pixelSpan[x].R, 0), index, 0, x, y);
+                                append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, x, y);
+                                append(tensor, pixToTensor(pixelSpan[x].B, 2), index, 2, x, y);
+                            }
+                        }
+                    }
+                }
+            });
+        }
+
         private static Tensor<float> InitInputTensor(ImagePreprocessorOptions options, int batchSize = 1)
         {
             switch (options.ChannelType)
@@ -127,24 +207,54 @@ namespace ZeroLevel.NN
             }
         }
 
-        public static PredictionInput[] ToTensors(this Image image, ImagePreprocessorOptions options)
+
+        public static ImagePredictionInput[] ToTensors(this Image image, ImagePreprocessorOptions options)
         {
-            PredictionInput[] result = null;
+            ImagePredictionInput[] result = null;
             var pixToTensor = PixelToTensorMethod(options);
             options.Channels = image.PixelType.BitsPerPixel >> 3;
-
             if (options.Crop.Enabled)
             {
-                var fragments = CalculateFragmentsCount(image, options);
+                // Размеры оригинального изображения
-                int count = CalculateFragmentsCount(image, options) + (options.Crop.SaveOriginal ? 1 : 0);
+                var Sw = image.Width;
+                var Sh = image.Height;
+
+                // Создание ресайза для целочисленного прохода кропами шириной CRw и высотой CRh
+                var resizedForCropWidthKoef = options.InputWidth / (double)options.Crop.Width;
+                var resizedForCropHeightKoef = options.InputHeight / (double)options.Crop.Height;
+
+                // Размеры для ресайза изображения к размеру по которому удобно идти кропами
+                var resizedForCropWidth = (int)Math.Round(Sw * resizedForCropWidthKoef, MidpointRounding.ToEven);
+                var resizedForCropHeight = (int)Math.Round(Sh * resizedForCropHeightKoef, MidpointRounding.ToEven);
+
+                // Размеры кропа, равны входу сети, а не (options.Crop.Width, options.Crop.Height), т.к. для оптимизации изображение будет предварительно отресайзено
+                var CRw = options.InputWidth;
+                var CRh = options.InputHeight;
+
+                // Расчет сдвигов между кропами
+                var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw;
+                var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh;
+
+                using (var source = image.Clone(img => img.Resize(resizedForCropWidth, resizedForCropHeight, KnownResamplers.Bicubic)))
+                {
+                    // Количество тензоров всего, во всех батчах суммарно
+                    var count = CalculateFragmentsCount(source, options);
+
+                    // Проверка, укладывается ли количество тензоров поровну в батчи
                     int offset = count % options.MaxBatchSize;
-                int count_tensors = count / options.MaxBatchSize + (offset == 0 ? 0 : 1);
+
-                var tensors = new PredictionInput[count_tensors];
+                    // Количество батчей
-                for (int i = 0; i < count_tensors; i++)
+                    int count_tensor_batches = count / options.MaxBatchSize + (offset == 0 ? 0 : 1);
+
+                    // Батчи
+                    var tensors = new ImagePredictionInput[count_tensor_batches];
+
+                    // Инициализация батчей
+                    Parallel.For(0, count_tensor_batches, batch_index =>
                     {
-                    if (i < count_tensors - 1)
+                        if (batch_index < count_tensor_batches - 1)
                         {
-                        tensors[i] = new PredictionInput
+                            tensors[batch_index] = new ImagePredictionInput
                             {
                                 Tensor = InitInputTensor(options, options.MaxBatchSize),
                                 Offsets = new OffsetBox[options.MaxBatchSize],
@@ -153,75 +263,83 @@ namespace ZeroLevel.NN
                         }
                         else
                         {
-                        tensors[i] = new PredictionInput
+                            tensors[batch_index] = new ImagePredictionInput
                             {
                                 Tensor = InitInputTensor(options, offset == 0 ? options.MaxBatchSize : offset),
                                 Offsets = new OffsetBox[offset == 0 ? options.MaxBatchSize : offset],
                                 Count = offset == 0 ? options.MaxBatchSize : offset
                             };
                         }
-                }
+                    });
 
+                    // Заполнение батчей
                     int tensor_index = 0;
-                int tensor_part_index = 0;
-                var xs = options.Crop.Overlap ? (int)(options.Crop.Width * options.Crop.OverlapKoefWidth) : options.Crop.Width;
-                var ys = options.Crop.Overlap ? (int)(options.Crop.Height * options.Crop.OverlapKoefHeight) : options.Crop.Height;
 
+                    // Если используется ресайз оригинала кроме кропов, пишется в первый батч в первый тензор
                     if (options.Crop.SaveOriginal)
                     {
-                    using (var copy = image.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
+                        using (var copy = source.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
                         {
-                        FillTensor(tensors[tensor_index].Tensor, copy, tensor_part_index, options, pixToTensor);
+                            FillTensor(tensors[0].Tensor, copy, 0, options, pixToTensor);
-                        tensors[tensor_index].Offsets[tensor_part_index] = new OffsetBox(0, 0, image.Width, image.Height);
+                            tensors[tensor_index].Offsets[0] = new OffsetBox(0, 0, image.Width, image.Height);
                         }
-                    tensor_part_index++;
+                        tensor_index++;
-                }
-                for (var x = 0; x < image.Width - xs; x += xs)
-                {
-                    var startx = x;
-                    var dx = (x + options.Crop.Width) - image.Width;
-                    if (dx > 0)
-                    {
-                        startx -= dx;
                     }
-                    for (var y = 0; y < image.Height - ys; y += ys)
+                    tensor_index--;
+                    Parallel.ForEach(SteppedIterator(0, source.Width, Dx), x =>
                     {
-                        if (tensor_part_index > 0 && tensor_part_index % options.MaxBatchSize == 0)
+                        // Можно запараллелить и тут, но выигрыш дает малоощутимый
+                        for (int y = 0; y < source.Height; y += Dy)
                         {
-                            tensor_index++;
+                            var current_index = Interlocked.Increment(ref tensor_index);
-                            tensor_part_index = 0;
+                            // Индекс тензора внутри батча
-                        }
+                            var b_index = current_index % options.MaxBatchSize;
-                        var starty = y;
+                            // Индекс батча
-                        var dy = (y + options.Crop.Height) - image.Height;
+                            var p_index = (int)Math.Round((double)current_index / (double)options.MaxBatchSize, MidpointRounding.ToNegativeInfinity);
-                        if (dy > 0)
+                            int w = CRw;
+                            if ((x + CRw) > source.Width)
                             {
-                            starty -= dy;
+                                w = source.Width - x;
                             }
-                        using (var copy = image
+                            int h = CRh;
-                            .Clone(img => img
+                            if ((y + CRh) > source.Height)
-                                .Crop(new Rectangle(startx, starty, options.Crop.Width, options.Crop.Height))
-                                .Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
                             {
-                            FillTensor(tensors[tensor_index].Tensor, copy, tensor_part_index, options, pixToTensor);
+                                h = source.Height - y;
-                            tensors[tensor_index].Offsets[tensor_part_index] = new OffsetBox(startx, starty, options.Crop.Width, options.Crop.Height);
-                        }
-                        tensor_part_index++;
                             }
+                            // Заполнение b_index тензора в p_index батче
+                            FillTensor(tensors[p_index].Tensor, source, x, y, w, h, b_index, options, pixToTensor);
+                            // Указание смещений для данного тензора
+                            tensors[p_index].Offsets[b_index] = new OffsetBox(x, y, options.Crop.Width, options.Crop.Height);
                         }
+                    });
                     return tensors;
                 }
-
+            }
             // if resize only
-            result = new PredictionInput[1];
+            result = new ImagePredictionInput[1];
             using (var copy = image.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
             {
                 Tensor<float> tensor = InitInputTensor(options);
                 FillTensor(tensor, copy, 0, options, pixToTensor);
-                result[0] = new PredictionInput { Count = 1, Offsets = null, Tensor = tensor };
+                result[0] = new ImagePredictionInput
+                {
+                    Count = 1,
+                    Offsets = null,
+                    Tensor = tensor
+                };
             }
             return result;
         }
 
+
+        private static IEnumerable<int> SteppedIterator(int startIndex, int endIndex, int stepSize)
+        {
+            for (int i = startIndex; i < endIndex; i += stepSize)
+            {
+                yield return i;
+            }
+        }
+
         public static Image Crop(Image source, float x1, float y1, float x2, float y2)
         {
             int left = 0;

ZeroLevel.NN/Services/NMS.cs

@@ -0,0 +1,118 @@
+using ZeroLevel.NN.Models;
+
+namespace ZeroLevel.NN.Services
+{
+    public static class NMS
+    {
+        private const float IOU_THRESHOLD = .2f;
+        private const float IOU_MERGE_THRESHOLD = .5f;
+
+        public static void Apply(List<YoloPrediction> boxes)
+        {
+            for (int i = 0; i < boxes.Count - 1; i++)
+            {
+                var left = boxes[i];
+                for (int j = i + 1; j < boxes.Count; j++)
+                {
+                    var right = boxes[j];
+                    if (left.Class == right.Class)
+                    {
+                        // удаление вложенных боксов
+                        var ni = NestingIndex(left, right);
+                        if (ni == 1)
+                        {
+                            boxes.RemoveAt(i);
+                            i--;
+                            break;
+                        }
+                        else if (ni == 2)
+                        {
+                            boxes.RemoveAt(j);
+                            j--;
+                        }
+                        // -----------------------------
+                        else
+                        {
+                            var iou = IOU(left, right);
+                            if (iou > IOU_THRESHOLD)
+                            {
+                                if (left.Score > right.Score)
+                                {
+                                    boxes.RemoveAt(j);
+                                    j--;
+                                }
+                                else
+                                {
+                                    boxes.RemoveAt(i);
+                                    i--;
+                                    break;
+                                }
+                            }
+                            /*else if (threshold > 0.01f && iou > 0.2f)
+                            {
+                                // UNION
+                                var x1 = Math.Min(left.X, right.X);
+                                var y1 = Math.Min(left.Y, right.Y);
+                                var x2 = Math.Max(left.X + left.W, right.X + right.W);
+                                var y2 = Math.Max(left.Y + left.H, right.Y + right.H);
+                                var w = x2 - x1;
+                                var h = y2 - y1;
+                                boxes.RemoveAt(j);
+                                boxes.RemoveAt(i);
+                                boxes.Add(new Prediction
+                                {
+                                    Class = left.Class,
+                                    Label = left.Label,
+                                    Score = Math.Max(left.Score, right.Score),
+                                    Cx = x1 + w / 2.0f,
+                                    Cy = y1 + h / 2.0f,
+                                    W = w,
+                                    H = h
+                                });
+                                i--;
+                                break;
+                            }*/
+                        }
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// проверка на вложенность боксов
+        /// </summary>
+        static int NestingIndex(YoloPrediction box1, YoloPrediction box2)
+        {
+            if (box1.X > box2.X &&
+                box1.Y > box2.Y &&
+                (box1.X + box1.W) < (box2.X + box2.W) &&
+                (box1.Y + box1.H) < (box2.Y + box2.H))
+            {
+                return 1;
+            }
+            if (box2.X > box1.X &&
+                box2.Y > box1.Y &&
+                (box2.X + box2.W) < (box1.X + box1.W) &&
+                (box2.Y + box2.H) < (box1.Y + box1.H))
+            {
+                return 2;
+            }
+            return 0;
+        }
+
+        static float IOU(YoloPrediction box1, YoloPrediction box2)
+        {
+            var left = (float)Math.Max(box1[0], box2[0]);
+            var right = (float)Math.Min(box1[2], box2[2]);
+
+            var top = (float)Math.Max(box1[1], box2[1]);
+            var bottom = (float)Math.Min(box1[3], box2[3]);
+
+            var width = right - left;
+            var height = bottom - top;
+            var intersectionArea = width * height;
+
+            return intersectionArea / (float)(box1.Area + box2.Area - intersectionArea);
+        }
+    }
+}

ZeroLevel.NN/Services/SSDNN.cs

@@ -1,7 +1,6 @@
 using Microsoft.ML.OnnxRuntime;
 using Microsoft.ML.OnnxRuntime.Tensors;
 using SixLabors.ImageSharp;
-using SixLabors.ImageSharp.PixelFormats;
 using ZeroLevel.NN.Models;
 
 namespace ZeroLevel.NN
@@ -18,6 +17,7 @@ namespace ZeroLevel.NN
                 try
                 {
                     var so = SessionOptions.MakeSessionOptionWithCudaProvider(0);
+                    so.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_VERBOSE;
                     so.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL;
                     _session = new InferenceSession(modelPath, so);
                 }
@@ -64,7 +64,7 @@ namespace ZeroLevel.NN
                 vector[i] *= inverseLength;
             }
         }
-        protected PredictionInput[] MakeInputBatch(Image<Rgb24> image, ImagePreprocessorOptions options)
+        protected ImagePredictionInput[] MakeInputBatch(Image image, ImagePreprocessorOptions options)
         {
             return ImagePreprocessor.ToTensors(image, options);
         }
@@ -75,6 +75,22 @@ namespace ZeroLevel.NN
             return input[0].Tensor;
         }
 
+        protected int Argmax(float[] embedding)
+        {
+            if (embedding.Length == 0) return -1;
+            var im = 0;
+            var max = embedding[0];
+            for (var i = 1; i < embedding.Length; i++)
+            {
+                if (embedding[i] > max)
+                {
+                    im = i;
+                    max = embedding[i];
+                }
+            }
+            return im;
+        }
+
         public void Dispose()
         {
             _session?.Dispose();

ZeroLevel.NN/ZeroLevel.NN.csproj

@@ -7,7 +7,7 @@
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
     <DebugType>embedded</DebugType>
     <ErrorReport>none</ErrorReport>
-    <Version>1.0.0.1</Version>
+    <Version>1.0.0.2</Version>
     <Company>Ogoun</Company>
     <Authors>Ogoun</Authors>
     <Copyright>Copyright Ogoun 2022</Copyright>
@@ -15,13 +15,18 @@
     <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
     <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
-    <PackageReleaseNotes>New architectures</PackageReleaseNotes>
+    <PackageReleaseNotes></PackageReleaseNotes>
+    <Platforms>AnyCPU;x64</Platforms>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
     <Optimize>False</Optimize>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Optimize>False</Optimize>
+  </PropertyGroup>
+
   <ItemGroup>
     <None Include="..\zero.png">
       <Pack>True</Pack>
@@ -30,10 +35,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Aurigma.GraphicsMill.Core.x64" Version="10.5.308" />
+    <PackageReference Include="Aurigma.GraphicsMill.Core.x64" Version="10.6.25" />
-    <PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.10.0" />
+    <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.11.0" />
-    <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.10.0" />
+    <PackageReference Include="SixLabors.ImageSharp" Version="2.1.3" />
-    <PackageReference Include="SixLabors.ImageSharp" Version="2.1.0" />
+    <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="1.0.0-beta14" />
   </ItemGroup>
 
   <ItemGroup>

ZeroLevel.SqLite/BaseSqLiteDB.cs

@@ -1,77 +1,94 @@
-using System;
+using SQLite;
-using System.Data.SQLite;
+using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq.Expressions;
 using ZeroLevel.Services.FileSystem;
 
 namespace ZeroLevel.SqLite
 {
-    public abstract class BaseSqLiteDB
+    public abstract class BaseSqLiteDB<T>
+        : IDisposable
+        where T : class, new()
     {
-        #region Helpers
+        protected SQLiteConnection _db;
-        protected static bool HasColumn(SQLiteDataReader dr, string columnName)
+        public BaseSqLiteDB(string name)
         {
-            for (int i = 0; i < dr.FieldCount; i++)
+            _db = new SQLiteConnection(PrepareDb(name));
-            {
-                if (dr.GetName(i).Equals(columnName, StringComparison.InvariantCultureIgnoreCase))
-                    return true;
-            }
-            return false;
         }
-        protected static Tr Read<Tr>(SQLiteDataReader reader, int index)
+
-        {
+        public T Append(T record)
-            if (reader == null || reader.FieldCount <= index || reader[index] == DBNull.Value) return default;
-            Type t;
-            if ((t = Nullable.GetUnderlyingType(typeof(Tr))) != null)
         {
-                return (Tr)Convert.ChangeType(reader[index], t);
+            _db.Insert(record);
+            return record;
         }
-            return (Tr)Convert.ChangeType(reader[index], typeof(Tr));
+
+        public void CreateTable()
+        {
+            _db.CreateTable<T>();
         }
-        protected static Tr Read<Tr>(SQLiteDataReader reader, string name)
+
+        public void DropTable()
         {
-            if (reader == null || !HasColumn(reader, name) || reader[name] == DBNull.Value) return default;
+            _db.DropTable<T>();
-            Type t;
+        }
-            if ((t = Nullable.GetUnderlyingType(typeof(Tr))) != null)
+
+        public IEnumerable<T> SelectAll()
         {
-                return (Tr)Convert.ChangeType(reader[name], t);
+            return _db.Table<T>();
         }
-            return (Tr)Convert.ChangeType(reader[name], typeof(Tr));
+
+        public IEnumerable<T> SelectBy(Expression<Func<T, bool>> predicate)
+        {
+            return _db.Table<T>().Where(predicate);
         }
 
-        protected static void Execute(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null)
+        public T Single(Expression<Func<T, bool>> predicate)
         {
-            using (var cmd = new SQLiteCommand(query, connection))
+            return _db.Table<T>().FirstOrDefault(predicate);
+        }
+
+        public T Single<U>(Expression<Func<T, bool>> predicate, Expression<Func<T, U>> orderBy, bool desc = false)
         {
-                if (parameters != null && parameters.Length > 0)
+            if (desc)
             {
-                    cmd.Parameters.AddRange(parameters);
+                return _db.Table<T>().Where(predicate).OrderByDescending(orderBy).FirstOrDefault();
-                }
-                cmd.ExecuteNonQuery();
             }
+            return _db.Table<T>().Where(predicate).OrderBy(orderBy).FirstOrDefault();
         }
 
-        protected static object ExecuteScalar(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null)
+        public T Single<U>(Expression<Func<T, U>> orderBy, bool desc = false)
-        {
-            using (var cmd = new SQLiteCommand(query, connection))
         {
-                if (parameters != null && parameters.Length > 0)
+            if (desc)
             {
-                    cmd.Parameters.AddRange(parameters);
+                return _db.Table<T>().OrderByDescending(orderBy).FirstOrDefault();
-                }
-                return cmd.ExecuteScalar();
             }
+            return _db.Table<T>().OrderBy(orderBy).FirstOrDefault();
         }
 
-        protected static SQLiteDataReader Read(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null)
+        public IEnumerable<T> SelectBy(int N, Expression<Func<T, bool>> predicate)
         {
-            using (var cmd = new SQLiteCommand(query, connection))
+            return _db.Table<T>().Where(predicate).Take(N);
+        }
+
+        public long Count()
         {
-                if (parameters != null && parameters.Length > 0)
+            return _db.Table<T>().Count();
+        }
+
+        public long Count(Expression<Func<T, bool>> predicate)
         {
-                    cmd.Parameters.AddRange(parameters);
+            return _db.Table<T>().Count(predicate);
         }
-                return cmd.ExecuteReader();
+
+        public int Delete(Expression<Func<T, bool>> predicate)
+        {
+            return _db.Table<T>().Delete(predicate);
         }
+
+        public void Update(T record)
+        {
+            _db.Update(record);
         }
 
         protected static string PrepareDb(string path)
@@ -80,13 +97,23 @@ namespace ZeroLevel.SqLite
             {
                 path = Path.Combine(FSUtils.GetAppLocalDbDirectory(), path);
             }
-            if (!File.Exists(path))
-            {
-                SQLiteConnection.CreateFile(path);
-            }
             return Path.GetFullPath(path);
         }
 
-        #endregion Helpers
+        protected abstract void DisposeStorageData();
+
+        public void Dispose()
+        {
+            DisposeStorageData();
+            try
+            {
+                _db?.Close();
+                _db?.Dispose();
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex, "[BaseSqLiteDB] Fault close db connection");
+            }            
+        }
     }
 }

ZeroLevel.SqLite/Model/User.cs

@@ -1,10 +1,15 @@
-namespace ZeroLevel.SqLite
+using SQLite;
+
+namespace ZeroLevel.SqLite
 {
     public class User
     {
+        [PrimaryKey, AutoIncrement]
         public long Id { get; set; }
+        [Indexed]
         public string UserName { get; set; }
         public string DisplayName { get; set; }
+        [Indexed]
         public byte[] PasswordHash { get; set; }
         public long Timestamp { get; set; }
         public long Creator { get; set; }

ZeroLevel.SqLite/SqLiteDelayDataStorage.cs

@@ -1,24 +1,28 @@
-using System;
+using SQLite;
-using System.Collections.Generic;
+using System;
-using System.Data.SQLite;
-using System.IO;
-using System.Text;
 using System.Threading;
 using ZeroLevel.Services.Serialization;
 using ZeroLevel.Services.Shedulling;
 
 namespace ZeroLevel.SqLite
 {
+    public sealed class ExpirationRecord
+    {
+        [PrimaryKey, AutoIncrement]
+        public long Id { get; set; }
+        [Indexed]
+        public long Expiration { get; set; }
+        public byte[] Data { get; set; }
+    }
+
     public sealed class SqLiteDelayDataStorage<T>
-        : BaseSqLiteDB, IDisposable
+        : BaseSqLiteDB<ExpirationRecord>
-        where T : IBinarySerializable
+        where T : class, IBinarySerializable, new()
     {
         #region Fields
 
         private readonly IExpirationSheduller _sheduller;
         private readonly Func<T, DateTime> _expire_date_calc_func;
-        private readonly SQLiteConnection _db;
-        private readonly string _table_name;
         private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
 
         #endregion Fields
@@ -28,14 +32,10 @@ namespace ZeroLevel.SqLite
         public SqLiteDelayDataStorage(string database_file_path,
             Func<T, bool> expire_callback,
             Func<T, DateTime> expire_date_calc_func)
+            : base(database_file_path)
         {
             this._expire_date_calc_func = expire_date_calc_func;
-            var path = PrepareDb(database_file_path);
+            CreateTable();
-            _table_name = "expiration";
-            _db = new SQLiteConnection($"Data Source={path};Version=3;");
-            _db.Open();
-            Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB, expirationtime INTEGER)", _db);
-            Execute($"CREATE INDEX IF NOT EXISTS expirationtime_index ON {_table_name} (expirationtime)", _db);
             _sheduller = Sheduller.CreateExpirationSheduller();
             OnExpire += expire_callback;
             Preload();
@@ -64,13 +64,8 @@ namespace ZeroLevel.SqLite
             long id = -1;
             try
             {
-                Execute($"INSERT INTO {_table_name} ('body', 'expirationtime') values (@body, @expirationtime)", _db,
+                var r = Append(new ExpirationRecord { Expiration = expirationTime, Data = MessageSerializer.Serialize(packet) });
-                    new SQLiteParameter[]
+                id = r.Id;
-                    {
-                        new SQLiteParameter("body", MessageSerializer.Serialize(packet)),
-                        new SQLiteParameter("expirationtime", expirationTime)
-                    });
-                id = (long)ExecuteScalar("select last_insert_rowid();", _db);
             }
             catch (Exception ex)
             {
@@ -91,17 +86,13 @@ namespace ZeroLevel.SqLite
 
         private void Preload()
         {
-            SQLiteDataReader reader;
             _rwLock.EnterReadLock();
             try
             {
-                reader = Read($"SELECT id, expirationtime FROM {_table_name}", _db);
+                foreach (var record in SelectAll())
-                while (reader.Read())
                 {
-                    var id = reader.GetInt64(0);
+                    _sheduller.Push(new DateTime(record.Expiration, DateTimeKind.Local), (k) => Pop(record.Id));
-                    _sheduller.Push(new DateTime(reader.GetInt64(1), DateTimeKind.Local), (k) => Pop(id));
                 }
-                reader.Close();
             }
             catch (Exception ex)
             {
@@ -111,7 +102,6 @@ namespace ZeroLevel.SqLite
             {
                 _rwLock.ExitReadLock();
             }
-            reader = null;
         }
 
         private void Pop(long id)
@@ -122,7 +112,7 @@ namespace ZeroLevel.SqLite
                 _rwLock.EnterReadLock();
                 try
                 {
-                    body = (byte[])ExecuteScalar($"SELECT body FROM {_table_name} WHERE id=@id", _db, new SQLiteParameter[] { new SQLiteParameter("id", id) });
+                    body = Single(r=>r.Id == id)?.Data;
                 }
                 catch (Exception ex)
                 {
@@ -161,8 +151,7 @@ namespace ZeroLevel.SqLite
             _rwLock.EnterWriteLock();
             try
             {
-                Execute($"DELETE FROM {_table_name} WHERE id = @id", _db,
+                Delete(r => r.Id == id);
-                    new SQLiteParameter[] { new SQLiteParameter("id", id) });
             }
             catch (Exception ex)
             {
@@ -178,17 +167,8 @@ namespace ZeroLevel.SqLite
 
         #region IDisposable
 
-        public void Dispose()
+        protected override void DisposeStorageData()
-        {
-            try
         {
-                _db?.Close();
-                _db?.Dispose();
-            }
-            catch (Exception ex)
-            {
-                Log.Error(ex, "[SqLiteDelayDataStorage] Fault close db connection");
-            }
             _sheduller.Dispose();
         }
 

ZeroLevel.SqLite/SqLiteDupStorage.cs

@@ -1,6 +1,6 @@
-using System;
+using SQLite;
+using System;
 using System.Collections.Generic;
-using System.Data.SQLite;
 using System.Linq;
 using System.Security.Cryptography;
 using System.Text;
@@ -8,40 +8,37 @@ using System.Threading;
 
 namespace ZeroLevel.SqLite
 {
+    public sealed class DuplicateRecord
+    {
+        [PrimaryKey, AutoIncrement]
+        public long Id { get; set; }
+        [Indexed]
+        public string Hash { get; set; }
+        public long Timestamp { get; set; }
+        public byte[] Data { get; set; }
+    }
+
     /// <summary>
     /// Хранит данные указанное число дней, и позволяет выполнить проверку наличия данных, для отбрасывания дубликатов
     /// </summary>
     public sealed class SqLiteDupStorage
-        : BaseSqLiteDB, IDisposable
+        : BaseSqLiteDB<DuplicateRecord>
     {
         #region Fields
 
-        private const string DEFAUL_TABLE_NAME = "History";
-
-        private readonly SQLiteConnection _db;
         private readonly long _removeOldRecordsTaskKey;
         private readonly int _countDays;
-        private readonly string _table_name;
         private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
 
         #endregion Fields
 
         #region Private members
-
-        private sealed class DuplicationStorageRecord
-        {
-            public string Hash { get; set; }
-            public byte[] Body { get; set; }
-            public long Timestamp { get; set; }
-        }
-
         private void RemoveOldRecordsTask(long key)
         {
             _rwLock.EnterWriteLock();
             try
             {
-                Execute($"DELETE FROM {_table_name} WHERE timestamp < @limit", _db,
+                Delete(r => r.Timestamp < DateTime.Now.AddDays(-_countDays).Ticks);
-                    new SQLiteParameter[] { new SQLiteParameter("limit", DateTime.Now.AddDays(-_countDays).Ticks) });
             }
             catch (Exception ex)
             {
@@ -57,21 +54,10 @@ namespace ZeroLevel.SqLite
 
         #region Ctor
 
-        public SqLiteDupStorage(string database_file_path, string tableName, int period)
+        public SqLiteDupStorage(string database_file_path, int period)
-        {
+            : base(database_file_path)
-            var path = PrepareDb(database_file_path);
-            if (string.IsNullOrWhiteSpace(tableName))
-            {
-                _table_name = DEFAUL_TABLE_NAME;
-            }
-            else
         {
-                _table_name = tableName;
+            CreateTable();
-            }
-            _db = new SQLiteConnection($"Data Source={path};Version=3;");
-            _db.Open();
-            Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, body BLOB, timestamp INTEGER)", _db);
-            Execute($"CREATE INDEX IF NOT EXISTS hash_index ON {_table_name} (hash)", _db);
             _countDays = period > 0 ? period : 1;
             _removeOldRecordsTaskKey = Sheduller.RemindEvery(TimeSpan.FromMinutes(1), RemoveOldRecordsTask);
         }
@@ -87,17 +73,14 @@ namespace ZeroLevel.SqLite
         {
             var hash = GenerateSHA256String(body);
             var timestamp = DateTime.Now.Ticks;
-            SQLiteDataReader reader;
             _rwLock.EnterReadLock();
             var exists = new List<byte[]>();
             try
             {
-                reader = Read($"SELECT body FROM {_table_name} WHERE hash=@hash", _db, new SQLiteParameter[] { new SQLiteParameter("hash", hash) });
+                foreach (var record in SelectBy(r => r.Hash == hash))
-                while (reader.Read())
                 {
-                    exists.Add((byte[])reader.GetValue(0));
+                    exists.Add(record.Data);
                 }
-                reader.Close();
             }
             catch (Exception ex)
             {
@@ -108,7 +91,6 @@ namespace ZeroLevel.SqLite
             {
                 _rwLock.ExitReadLock();
             }
-            reader = null;
             if (exists.Any())
             {
                 foreach (var candidate in exists)
@@ -120,12 +102,11 @@ namespace ZeroLevel.SqLite
             _rwLock.EnterWriteLock();
             try
             {
-                Execute($"INSERT INTO {_table_name} ('hash', 'body', 'timestamp') values (@hash, @body, @timestamp)", _db,
+                Append(new DuplicateRecord
-                    new SQLiteParameter[]
                 {
-                        new SQLiteParameter("hash", hash),
+                    Data = body,
-                        new SQLiteParameter("body", body),
+                    Hash = hash,
-                        new SQLiteParameter("timestamp", timestamp)
+                    Timestamp = timestamp
                 });
             }
             catch (Exception ex)
@@ -143,17 +124,9 @@ namespace ZeroLevel.SqLite
 
         #region IDisposable
 
-        public void Dispose()
+        protected override void DisposeStorageData()
         {
             Sheduller.Remove(_removeOldRecordsTaskKey);
-            try
-            {
-                _db?.Dispose();
-            }
-            catch (Exception ex)
-            {
-                Log.Error(ex, "[SQLiteDupStorage] Fault close db connection");
-            }
         }
 
         #endregion IDisposable

ZeroLevel.SqLite/SqLitePacketBuffer.cs

@@ -1,15 +1,25 @@
-using System;
+using SQLite;
-using System.Data.SQLite;
+using System;
 using System.Threading;
 using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel.SqLite
 {
+
+    public sealed class PacketRecord
+    {
+        [PrimaryKey, AutoIncrement]
+        public long Id { get; set; }
+        [Indexed]
+        public long Timestamp { get; set; }
+        public byte[] Data { get; set; }
+    }
+
     /// <summary>
     /// Промежуточное/временное хранилище пакетов данных, для случаев сбоя доставок через шину данных
     /// </summary>
     public sealed class SqLitePacketBuffer<T>
-        : BaseSqLiteDB, IDisposable
+        : BaseSqLiteDB<PacketRecord>
         where T : IBinarySerializable
     {
         private sealed class PacketBufferRecord
@@ -20,20 +30,14 @@ namespace ZeroLevel.SqLite
 
         #region Fields
 
-        private readonly SQLiteConnection _db;
-        private readonly string _table_name;
         private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
 
         #endregion Fields
 
         public SqLitePacketBuffer(string database_file_path)
+            : base(database_file_path)
         {
-            var path = PrepareDb(database_file_path);
+            CreateTable();
-            _table_name = "packets";
-            _db = new SQLiteConnection($"Data Source={path};Version=3;");
-            _db.Open();
-            Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB, created INTEGER)", _db);
-            Execute($"CREATE INDEX IF NOT EXISTS created_index ON {_table_name} (created)", _db);
         }
 
         public void Push(T frame)
@@ -43,13 +47,11 @@ namespace ZeroLevel.SqLite
             var creationTime = DateTime.Now.Ticks;
             try
             {
-                Execute($"INSERT INTO {_table_name} ('body', 'created') values (@body, @created)", _db,
+                id = Append(new PacketRecord 
-                    new SQLiteParameter[]
                 {
-                        new SQLiteParameter("body", MessageSerializer.Serialize(frame)),
+                    Data = MessageSerializer.Serialize(frame),
-                        new SQLiteParameter("created", creationTime)
+                    Timestamp = creationTime
-                    });
+                }).Id;                
-                id = (long)ExecuteScalar("select last_insert_rowid();", _db);
             }
             catch (Exception ex)
             {
@@ -65,15 +67,12 @@ namespace ZeroLevel.SqLite
         {
             bool success = false;
             long id = -1;
-            SQLiteDataReader reader;
             _rwLock.EnterReadLock();
             try
             {
-                reader = Read($"SELECT id, body FROM {_table_name} ORDER BY created ASC LIMIT 1", _db);
+                var record = Single(r => r.Timestamp);
-                if (reader.Read())
+                id = record.Id;
-                {
+                var body = record.Data;
-                    id = reader.GetInt64(0);
-                    var body = (byte[])reader.GetValue(1);
                 try
                 {
                     success = pop_callback(MessageSerializer.Deserialize<T>(body));
@@ -83,8 +82,6 @@ namespace ZeroLevel.SqLite
                     Log.Error(ex, "Fault handle buffered data");
                 }
             }
-                reader.Close();
-            }
             catch (Exception ex)
             {
                 Log.Error(ex, "[SqLitePacketBuffer] Fault preload datafrom db");
@@ -97,7 +94,6 @@ namespace ZeroLevel.SqLite
             {
                 RemoveRecordById(id);
             }
-            reader = null;
             return success;
         }
 
@@ -106,8 +102,7 @@ namespace ZeroLevel.SqLite
             _rwLock.EnterWriteLock();
             try
             {
-                Execute($"DELETE FROM {_table_name} WHERE id = @id", _db,
+                Delete(r => r.Id == id);
-                    new SQLiteParameter[] { new SQLiteParameter("id", id) });
             }
             catch (Exception ex)
             {
@@ -119,17 +114,8 @@ namespace ZeroLevel.SqLite
             }
         }
 
-        public void Dispose()
+        protected override void DisposeStorageData()
         {
-            try
-            {
-                _db?.Close();
-                _db?.Dispose();
-            }
-            catch (Exception ex)
-            {
-                Log.Error(ex, "[SqLitePacketBuffer] Fault close db connection");
-            }
         }
     }
 }

ZeroLevel.SqLite/SqLiteUserRepository.cs

@@ -1,18 +1,15 @@
 using System;
 using System.Collections.Generic;
-using System.Data.SQLite;
 using System.Threading;
 using ZeroLevel.Models;
 
 namespace ZeroLevel.SqLite
 {
     public class SqLiteUserRepository
-        : BaseSqLiteDB
+        : BaseSqLiteDB<User>
     {
         #region Fields
 
-        private readonly SQLiteConnection _db;
-        private readonly string _table_name = "users";
         private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
 
         #endregion Fields
@@ -20,13 +17,8 @@ namespace ZeroLevel.SqLite
         #region Ctor
 
         public SqLiteUserRepository()
+            : base("users.db")
         {
-            var path =PrepareDb("users.db");
-            _db = new SQLiteConnection($"Data Source={path};Version=3;");
-            _db.Open();
-            Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT, displayname TEXT, hash BLOB, timestamp INTEGER, creator INTEGER, role INTEGER)", _db);
-            Execute($"CREATE INDEX IF NOT EXISTS username_index ON {_table_name} (username)", _db);
-            Execute($"CREATE INDEX IF NOT EXISTS hash_index ON {_table_name} (hash)", _db);
         }
 
         #endregion Ctor
@@ -34,25 +26,13 @@ namespace ZeroLevel.SqLite
         public IEnumerable<User> GetAll()
         {
             var list = new List<User>();
-            SQLiteDataReader reader;
             _rwLock.EnterReadLock();
             try
             {
-                reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name}", _db);
+                foreach (var r in SelectAll())
-                while (reader.Read())
                 {
-                    list.Add(new User
+                    list.Add(r);
-                    {
-                        Id = reader.GetInt64(0),
-                        UserName = reader.GetString(1),
-                        DisplayName = Read<string>(reader, 2),
-                        PasswordHash = (byte[])reader.GetValue(3),
-                        Timestamp = reader.GetInt64(4),
-                        Creator = reader.GetInt64(5),
-                        Role = (UserRole)reader.GetInt32(6)
-                    });
                 }
-                reader.Close();
             }
             catch (Exception ex)
             {
@@ -62,34 +42,16 @@ namespace ZeroLevel.SqLite
             {
                 _rwLock.ExitReadLock();
             }
-            reader = null;
             return list;
         }
 
         public User Get(long id)
         {
             User user = null;
-            SQLiteDataReader reader;
             _rwLock.EnterReadLock();
             try
             {
-                reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name} WHERE id = @id", _db,
+                user = Single(r => r.Id == id);
-                    new SQLiteParameter[] { new SQLiteParameter("id", id) });
-                if (reader.Read())
-                {
-                    var body = (byte[])reader.GetValue(1);
-                    user = new User
-                    {
-                        Id = reader.GetInt64(0),
-                        UserName = reader.GetString(1),
-                        DisplayName = reader.GetString(2),
-                        PasswordHash = (byte[])reader.GetValue(3),
-                        Timestamp = reader.GetInt64(4),
-                        Creator = reader.GetInt64(5),
-                        Role = (UserRole)reader.GetInt32(6)
-                    };
-                }
-                reader.Close();
             }
             catch (Exception ex)
             {
@@ -99,37 +61,16 @@ namespace ZeroLevel.SqLite
             {
                 _rwLock.ExitReadLock();
             }
-            reader = null;
             return user;
         }
 
         public User Get(string username, byte[] hash)
         {
             User user = null;
-            SQLiteDataReader reader;
             _rwLock.EnterReadLock();
             try
             {
-                reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name} WHERE username = @username AND hash = @hash", _db,
+                user = Single(r => r.UserName == username && r.PasswordHash == hash);
-                    new SQLiteParameter[]
-                    {
-                        new SQLiteParameter("username", username),
-                        new SQLiteParameter("hash", hash)
-                    });
-                if (reader.Read())
-                {
-                    user = new User
-                    {
-                        Id = reader.GetInt64(0),
-                        UserName = reader.GetString(1),
-                        DisplayName = reader.GetString(2),
-                        PasswordHash = (byte[])reader.GetValue(3),
-                        Timestamp = reader.GetInt64(4),
-                        Creator = reader.GetInt64(5),
-                        Role = (UserRole)reader.GetInt32(6)
-                    };
-                }
-                reader.Close();
             }
             catch (Exception ex)
             {
@@ -139,7 +80,6 @@ namespace ZeroLevel.SqLite
             {
                 _rwLock.ExitReadLock();
             }
-            reader = null;
             return user;
         }
 
@@ -150,22 +90,12 @@ namespace ZeroLevel.SqLite
             var creationTime = DateTime.UtcNow.Ticks;
             try
             {
-                var count_obj = ExecuteScalar($"SELECT COUNT(*) FROM {_table_name} WHERE username=@username", _db, new SQLiteParameter[] { new SQLiteParameter("username", user.UserName) });
+                var count_obj = Count(r => r.UserName == user.UserName);
-                if (count_obj != null && (long)count_obj > 0)
+                if (count_obj > 0)
                 {
-                    return InvokeResult<long>.Fault<long>("Пользователь уже существует");
+                    return InvokeResult<long>.Fault<long>("Пользователь с таким именем уже существует");
                 }
-                Execute($"INSERT INTO {_table_name} ('username', 'displayname', 'hash', 'timestamp', 'creator', 'role') values (@username, @displayname, @hash, @timestamp, @creator, @role)", _db,
+                id = Append(user).Id;
-                    new SQLiteParameter[]
-                    {
-                        new SQLiteParameter("username", user.UserName),
-                        new SQLiteParameter("displayname", user.DisplayName),
-                        new SQLiteParameter("hash", user.PasswordHash),
-                        new SQLiteParameter("timestamp", creationTime),
-                        new SQLiteParameter("creator", user.Creator),
-                        new SQLiteParameter("role", user.Role)
-                    });
-                id = (long)ExecuteScalar("select last_insert_rowid();", _db);
             }
             catch (Exception ex)
             {
@@ -184,8 +114,7 @@ namespace ZeroLevel.SqLite
             _rwLock.EnterWriteLock();
             try
             {
-                Execute($"DELETE FROM {_table_name} WHERE username = @username", _db,
+                Delete(r => r.UserName == login);
-                    new SQLiteParameter[] { new SQLiteParameter("username", login) });
                 return InvokeResult.Succeeding();
             }
             catch (Exception ex)
@@ -198,5 +127,9 @@ namespace ZeroLevel.SqLite
                 _rwLock.ExitWriteLock();
             }
         }
+
+        protected override void DisposeStorageData()
+        {
+        }
     }
 }

ZeroLevel.SqLite/UserCacheRepository.cs

@@ -1,36 +1,36 @@
-using System;
+using SQLite;
+using System;
 using System.Collections.Generic;
-using System.Data.SQLite;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel.SqLite
 {
+    public sealed class DataRecord
+    {
+        [PrimaryKey, AutoIncrement]
+        public long Id { get; set; }
+        [Indexed]
+        public string Key { get; set; }
+        public byte[] Data { get; set; }
+    }
     public sealed class UserCacheRepository<T>
-         : BaseSqLiteDB, IDisposable
+         : BaseSqLiteDB<DataRecord>
              where T : IBinarySerializable
     {
         #region Fields
 
-        private readonly SQLiteConnection _db;
         private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
-        private readonly string _tableName;
 
         #endregion Fields
 
         #region Ctor
 
         public UserCacheRepository()
+            : base(typeof(T).Name)
         {
-            _tableName = typeof(T).Name;
+            CreateTable();
-
-            var path = PrepareDb($"{_tableName}_user_cachee.db");
-            _db = new SQLiteConnection($"Data Source={path};Version=3;");
-            _db.Open();
-
-            Execute($"CREATE TABLE IF NOT EXISTS {_tableName} (id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT, body BLOB)", _db);
-            Execute($"CREATE INDEX IF NOT EXISTS key_index ON {_tableName} (key)", _db);
         }
 
         #endregion Ctor
@@ -47,8 +47,8 @@ namespace ZeroLevel.SqLite
             _rwLock.EnterReadLock();
             try
             {
-                var count_obj = ExecuteScalar($"SELECT COUNT(*) FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[] { new SQLiteParameter("key", key) });
+                var count_obj = Count(r => r.Key == key);
-                if (count_obj != null && (long)count_obj > 0)
+                if (count_obj > 0)
                 {
                     update = true;
                 }
@@ -68,20 +68,16 @@ namespace ZeroLevel.SqLite
                 var body = MessageSerializer.Serialize(data);
                 if (update)
                 {
-                    Execute($"UPDATE {_tableName} SET body=@body WHERE key=@key", _db,
+                    var r = Single(r => r.Key == key);
-                    new SQLiteParameter[]
+                    r.Data = body;
-                    {
+                    Update(r);
-                            new SQLiteParameter("key", key),
-                            new SQLiteParameter("body", body)
-                    });
                 }
                 else
                 {
-                    Execute($"INSERT INTO {_tableName} ('key', 'body') values (@key, @body)", _db,
+                    Append(new DataRecord
-                    new SQLiteParameter[]
                     {
-                            new SQLiteParameter("key", key),
+                        Data = body,
-                            new SQLiteParameter("body", body)
+                        Key = key
                     });
                 }
                 return true;
@@ -103,10 +99,7 @@ namespace ZeroLevel.SqLite
             _rwLock.EnterWriteLock();
             try
             {
-                Execute($"DELETE FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[]
+                Delete(r => r.Key == key);
-                {
-                        new SQLiteParameter("key", key)
-                });
             }
             catch (Exception ex)
             {
@@ -124,10 +117,7 @@ namespace ZeroLevel.SqLite
             _rwLock.EnterWriteLock();
             try
             {
-                return Convert.ToInt64(ExecuteScalar($"SELECT COUNT(*) FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[]
+                return Count(r => r.Key == key);
-                {
-                        new SQLiteParameter("key", key)
-                }));
             }
             catch (Exception ex)
             {
@@ -144,23 +134,17 @@ namespace ZeroLevel.SqLite
         {
             var key = KEY(userid, name);
             var result = new List<T>();
-            SQLiteDataReader reader;
             _rwLock.EnterReadLock();
             try
             {
-                reader = Read($"SELECT [body] FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[]
+                foreach (var r in SelectBy(r=>r.Key == key))
                 {
-                        new SQLiteParameter("key", key)
+                    var data = r.Data;
-                });
-                while (reader.Read())
-                {
-                    var data = Read<byte[]>(reader, 0);
                     if (data != null)
                     {
                         result.Add(MessageSerializer.Deserialize<T>(data));
                     }
                 }
-                reader.Close();
             }
             catch (Exception ex)
             {
@@ -169,7 +153,6 @@ namespace ZeroLevel.SqLite
             finally
             {
                 _rwLock.ExitReadLock();
-                reader = null;
             }
             return result;
         }
@@ -178,16 +161,8 @@ namespace ZeroLevel.SqLite
 
         #region IDisposable
         
-        public void Dispose()
+        protected override void DisposeStorageData()
-        {
-            try
         {
-                _db?.Dispose();
-            }
-            catch (Exception ex)
-            {
-                Log.Error(ex, "[UserCacheRepository] Fault close db connection");
-            }
         }
 
         #endregion IDisposable

ZeroLevel.SqLite/ZeroLevel.SqLite.csproj

@@ -13,9 +13,9 @@ Based on System.Data.SQLite.Core</Description>
     <PackageIconUrl />
     <Platforms>AnyCPU;x64;x86</Platforms>
     <PackageReleaseNotes>Move to new ZeroLevel version</PackageReleaseNotes>
-    <AssemblyVersion>1.0.3.0</AssemblyVersion>
+    <AssemblyVersion>1.0.4.0</AssemblyVersion>
-    <FileVersion>1.0.3.0</FileVersion>
+    <FileVersion>1.0.4.0</FileVersion>
-    <Version>1.0.3.0</Version>
+    <Version>1.0.4.0</Version>
     <OutputType>Library</OutputType>
   </PropertyGroup>
 
@@ -27,11 +27,17 @@ Based on System.Data.SQLite.Core</Description>
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="System.Data.SQLite.Core" Version="1.0.115" />
+    <PackageReference Include="sqlite-net-static" Version="1.8.116" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\ZeroLevel\ZeroLevel.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <None Update="sqlite3.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>

ZeroLevel.UnitTests/SuffixAutomataTests.cs

@@ -0,0 +1,42 @@
+using Xunit;
+using ZeroLevel.Semantic;
+using ZeroLevel.Services.Text;
+
+namespace ZeroLevel.UnitTests
+{
+    public class SuffixAutomataTests
+    {
+        [Fact]
+        public void IsSubstringTest()
+        {
+            var text = @"=Однако эту оценку легко показать и без знания алгоритма. Вспомним о том, что число состояний равно количеству различных значений множеств endpos.#";
+            var automata = new SuffixAutomata();
+            automata.Init();
+            foreach (var ch in text)
+            {
+                automata.Extend(ch);
+            }
+            Assert.True(automata.IsSubstring("Вспомним"));
+            Assert.True(automata.IsSubstring("")); // empty line
+            Assert.True(automata.IsSubstring("#")); // end line
+            Assert.True(automata.IsSubstring("=")); // start line
+            Assert.True(automata.IsSubstring(null)); // null
+            Assert.False(automata.IsSubstring("равноценно"));
+            Assert.False(automata.IsSubstring("нетслова"));
+        }
+
+        [Fact]
+        public void IntersectionTest()
+        {
+            var text = @"=Однако эту оценку легко показать и без знания алгоритма. Вспомним о том, что число состояний равно количеству различных значений множеств endpos.#";
+            var i = LongestCommonSubstring.LCS(text, "енк");
+            Assert.Equal("енк", i);
+
+            i = LongestCommonSubstring.LCS(text, "стоя");
+            Assert.Equal("стоя", i);
+
+            i = LongestCommonSubstring.LCS(text, "горизонт");
+            Assert.Equal("гори", i);
+        }
+    }
+}

ZeroLevel.sln

@@ -39,8 +39,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "WPFExamples", "WPFExamples"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ConnectionTest", "ConnectionTest", "{D5207A5A-2F27-4992-9BA5-0BDCFC59F133}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Client", "ConnectionTest\Client\Client.csproj", "{08CDD42E-E324-40A4-88C3-EDD0493AAF84}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Server", "ConnectionTest\Server\Server.csproj", "{3496A688-0749-48C2-BD60-ABB42A5C17C9}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ZeroLevel.Qdrant", "ZeroLevel.Qdrant\ZeroLevel.Qdrant.csproj", "{7188B89E-96EB-4EFB-AAFB-D0A823031F99}"
@@ -63,7 +61,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Source", "TestPipeLine\Sour
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Watcher", "TestPipeLine\Watcher\Watcher.csproj", "{F70842E7-9A1D-4CC4-9F55-0953AEC9C7C8}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ZeroLevel.NN", "ZeroLevel.NN\ZeroLevel.NN.csproj", "{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ZeroLevel.NN", "ZeroLevel.NN\ZeroLevel.NN.csproj", "{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Client", "ConnectionTest\Client\Client.csproj", "{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AutoLoader", "AutoLoader", "{2EF83101-63BC-4397-A005-A747189143D4}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AppContainer", "AutoLoader\AppContainer\AppContainer.csproj", "{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -171,18 +175,6 @@ Global
 		{04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x64.Build.0 = Release|x64
 		{04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x86.ActiveCfg = Release|x86
 		{04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x86.Build.0 = Release|x86
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x64.ActiveCfg = Debug|x64
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x64.Build.0 = Debug|x64
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x86.ActiveCfg = Debug|x86
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x86.Build.0 = Debug|x86
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|Any CPU.Build.0 = Release|Any CPU
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x64.ActiveCfg = Release|x64
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x64.Build.0 = Release|x64
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x86.ActiveCfg = Release|x86
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x86.Build.0 = Release|x86
 		{3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|x64.ActiveCfg = Debug|x64
@@ -327,12 +319,35 @@ Global
 		{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x64.Build.0 = Release|Any CPU
 		{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x86.ActiveCfg = Release|Any CPU
 		{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x86.Build.0 = Release|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x64.Build.0 = Debug|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x86.Build.0 = Debug|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x64.ActiveCfg = Release|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x64.Build.0 = Release|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x86.ActiveCfg = Release|Any CPU
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x86.Build.0 = Release|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x64.Build.0 = Debug|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x86.Build.0 = Debug|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x64.ActiveCfg = Release|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x64.Build.0 = Release|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x86.ActiveCfg = Release|Any CPU
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
-		{08CDD42E-E324-40A4-88C3-EDD0493AAF84} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133}
 		{3496A688-0749-48C2-BD60-ABB42A5C17C9} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133}
 		{54BB2BA9-DAC0-4162-8BC0-E4A9B898CBB0} = {FC074553-5D9F-4DF1-9130-7092E37DE768}
 		{3D0FE0BA-F7B1-4A63-BBA4-C96514A68426} = {FC074553-5D9F-4DF1-9130-7092E37DE768}
@@ -340,6 +355,8 @@ Global
 		{5CCFF557-C91F-4DD7-9530-D76FE517DA98} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15}
 		{82202433-6426-4737-BAB2-473AC1F74C5D} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15}
 		{F70842E7-9A1D-4CC4-9F55-0953AEC9C7C8} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15}
+		{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133}
+		{9DE345EA-955B-41A8-93AF-277C0B5A9AC5} = {2EF83101-63BC-4397-A005-A747189143D4}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A65DB16F-877D-4586-A9F3-8BBBFBAF5CEB}

ZeroLevel/Services/Extensions/LinqExtension.cs

@@ -4,6 +4,21 @@ namespace System.Linq
 {
     public static class LinqExtension
     {
+        public static void ThrowIfNull<T>(this T val, string message = null)
+        {
+            if (null == val)
+                throw new ArgumentNullException(message);
+        }
+
+        public static void Apply<T>(this IEnumerable<T> seq, Action<T> action)
+        {
+            seq.ThrowIfNull();
+            action.ThrowIfNull();
+            foreach (var e in seq)
+            {
+                action.Invoke(e);
+            }
+        }
         public static IEnumerable<T[]> ZipLongest<T>(this IEnumerable<T> left, IEnumerable<T> right)
         {
             IEnumerator<T> leftEnumerator = left.GetEnumerator();

ZeroLevel/Services/FileSystem/FSUtils.cs

@@ -341,6 +341,27 @@ namespace ZeroLevel.Services.FileSystem
             var zipFile = Path.Combine(tmp.FullName, "zip.zip");
             File.WriteAllBytes(zipFile, data);
             ZipFile.ExtractToDirectory(zipFile, targetFolder);
+            File.Delete(zipFile);
+        }
+
+        public static void UnPackFolder(string zipFile, string targetFolder)
+        {
+            if (Directory.Exists(targetFolder))
+            {
+                try
+                {
+                    FSUtils.RemoveFolder(targetFolder, 3, 3000);
+                }
+                catch (Exception ex)
+                {
+                    Log.SystemError(ex, $"[FSUtils] Fault clean folder '{Path.GetDirectoryName(targetFolder)}'");
+                }
+            }
+            if (Directory.Exists(targetFolder) == false)
+            {
+                Directory.CreateDirectory(targetFolder);
+            }
+            ZipFile.ExtractToDirectory(zipFile, targetFolder);
         }
 
         public static void CopyDir(string sourceFolder, string targetFolder)

ZeroLevel/Services/HashFunctions/StringHash.cs

@@ -6,13 +6,14 @@
     /// </summary>
     public static class StringHash
     {
-        public static uint DotNetFullHash(string str)
+        public static int DotNetFullHash(string str)
         {
             unchecked
             {
                 int hash1 = (5381 << 16) + 5381;
                 int hash2 = hash1;
-
+                if (str != null)
+                {
                     for (int i = 0; i < str.Length; i += 2)
                     {
                         hash1 = ((hash1 << 5) + hash1) ^ str[i];
@@ -20,8 +21,8 @@
                             break;
                         hash2 = ((hash2 << 5) + hash2) ^ str[i + 1];
                     }
-
+                }
-                return (uint)(hash1 + (hash2 * 1566083941)) & 0x7FFFFFFF;
+                return (hash1 + (hash2 * 1566083941)) & 0x7FFFFFFF;
             }
         }
 

ZeroLevel/Services/MemoryPools/JetPool.cs

@@ -15,4 +15,15 @@ namespace MemoryPools
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void Return(T instance) => _freeObjectsQueue.Push(instance);
     }
+
+    public class JetValPool<T>
+    {
+        private readonly JetStack<T> _freeObjectsQueue = new JetStack<T>();
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public T Get() => _freeObjectsQueue.Count > 0 ? _freeObjectsQueue.Pop() : default(T);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Return(T instance) => _freeObjectsQueue.Push(instance);
+    }
 }

ZeroLevel/Services/Network/Contracts/ISocketClient.cs

@@ -13,8 +13,8 @@ namespace ZeroLevel.Network
 
         IRouter Router { get; }
 
-        bool Send(Frame data);
+        void Send(Frame data);
-        bool Request(Frame data, Action<byte[]> callback, Action<string> fail = null);
+        void Request(Frame data, Action<byte[]> callback, Action<string> fail = null);
-        bool Response(byte[] data, int identity);
+        void Response(byte[] data, int identity);
     }
 }

ZeroLevel/Services/Network/ExClient.cs

@@ -22,7 +22,8 @@ namespace ZeroLevel.Network
         {
             try
             {
-                return _client.Send(FrameFactory.Create(inbox));
+                _client.Send(FrameFactory.Create(inbox));
+                return true;
             }
             catch (Exception ex)
             {
@@ -35,7 +36,8 @@ namespace ZeroLevel.Network
         {
             try
             {
-                return _client.Send(FrameFactory.Create(inbox, data));
+                _client.Send(FrameFactory.Create(inbox, data));
+                return true;
             }
             catch (Exception ex)
             {
@@ -48,7 +50,8 @@ namespace ZeroLevel.Network
         {
             try
             {                
-                return _client.Send(FrameFactory.Create(BaseSocket.DEFAULT_MESSAGE_INBOX, MessageSerializer.SerializeCompatible<T>(message)));
+                _client.Send(FrameFactory.Create(BaseSocket.DEFAULT_MESSAGE_INBOX, MessageSerializer.SerializeCompatible<T>(message)));
+                return true;
             }
             catch (Exception ex)
             {
@@ -61,7 +64,8 @@ namespace ZeroLevel.Network
         {
             try
             {                
-                return _client.Send(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible<T>(message)));
+                _client.Send(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible<T>(message)));
+                return true;
             }
             catch (Exception ex)
             {
@@ -74,7 +78,8 @@ namespace ZeroLevel.Network
         {
             try
             {                
-                return _client.Request(FrameFactory.Create(inbox), f => callback(f));
+                _client.Request(FrameFactory.Create(inbox), f => callback(f));
+                return true;
             }
             catch (Exception ex)
             {
@@ -87,7 +92,8 @@ namespace ZeroLevel.Network
         {
             try
             {
-                return _client.Request(FrameFactory.Create(inbox, data), f => callback(f));
+                _client.Request(FrameFactory.Create(inbox, data), f => callback(f));
+                return true;
             }
             catch (Exception ex)
             {
@@ -100,7 +106,8 @@ namespace ZeroLevel.Network
         {
             try
             {
-                return _client.Request(FrameFactory.Create(inbox), f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
+                _client.Request(FrameFactory.Create(inbox), f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
+                return true;
             }
             catch (Exception ex)
             {
@@ -113,7 +120,8 @@ namespace ZeroLevel.Network
         {
             try
             {
-                return _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_INBOX), f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
+                _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_INBOX), f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
+                return true;
             }
             catch (Exception ex)
             {
@@ -126,8 +134,9 @@ namespace ZeroLevel.Network
         {
             try
             {
-                return _client.Request(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible<Trequest>(request)),
+                _client.Request(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible<Trequest>(request)),
                     f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
+                return true;
             }
             catch (Exception ex)
             {
@@ -140,8 +149,9 @@ namespace ZeroLevel.Network
         {
             try
             {
-                return _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_WITHOUT_ARGS_INBOX, MessageSerializer.SerializeCompatible<Trequest>(request)),
+                _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_WITHOUT_ARGS_INBOX, MessageSerializer.SerializeCompatible<Trequest>(request)),
                     f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
+                return true;
             }
             catch (Exception ex)
             {

ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs

@@ -1,7 +1,6 @@
 using MemoryPools;
 using System;
 using System.Threading;
-using ZeroLevel.Services.Pools;
 
 namespace ZeroLevel.Network
 {

ZeroLevel/Services/Network/SocketClient.cs

@@ -1,4 +1,5 @@
-using System;
+using MemoryPools;
+using System;
 using System.Collections.Concurrent;
 using System.Net;
 using System.Net.Sockets;
@@ -19,8 +20,17 @@ namespace ZeroLevel.Network
             public int identity;
             public byte[] data;
         }
+        private struct OutcomingFrame
+        {
+            public bool is_request;
+            public int identity;
+            public byte[] data;
+        }
 
-        private BlockingCollection<IncomingFrame> _incoming_queue = new BlockingCollection<IncomingFrame>();
+        private readonly JetValPool<OutcomingFrame> _outcomingFramesPool = new JetValPool<OutcomingFrame>();
+        private ConcurrentQueue<IncomingFrame> _incoming_queue = new ConcurrentQueue<IncomingFrame>();
+        private ConcurrentQueue<OutcomingFrame> _outcoming_queue = new ConcurrentQueue<OutcomingFrame>();
+        private ManualResetEventSlim _outcomingFrameEvent = new ManualResetEventSlim(false);
         #endregion
 
         private Socket _clientSocket;
@@ -31,6 +41,7 @@ namespace ZeroLevel.Network
         private readonly object _reconnection_lock = new object();
         private long _heartbeat_key;
         private Thread _receiveThread;
+        private Thread _sendingThread;
 
         #endregion Private
 
@@ -41,7 +52,6 @@ namespace ZeroLevel.Network
             try
             {
                 _clientSocket = new Socket(ep.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
-                _clientSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontLinger, true);
                 _clientSocket.Connect(ep);
                 OnConnect(this);
             }
@@ -81,6 +91,11 @@ namespace ZeroLevel.Network
             _receiveThread.IsBackground = true;
             _receiveThread.Start();
 
+            _sendingThread = new Thread(OutcomingFramesJob);
+            _sendingThread.IsBackground = true;
+            _sendingThread.Start();
+
+
             _heartbeat_key = Sheduller.RemindEvery(TimeSpan.FromMilliseconds(MINIMUM_HEARTBEAT_UPDATE_PERIOD_MS), Heartbeat);
         }
 
@@ -109,27 +124,26 @@ namespace ZeroLevel.Network
         public event Action<ISocketClient> OnDisconnect = (_) => { };
         public IPEndPoint Endpoint { get; }
 
-        public bool Request(Frame frame, Action<byte[]> callback, Action<string> fail = null)
+        public void Request(Frame frame, Action<byte[]> callback, Action<string> fail = null)
         {
             if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Request] Socket status: {Status}");
             var data = NetworkPacketFactory.Reqeust(MessageSerializer.Serialize(frame), out int id);
             _requests.RegisterForFrame(id, callback, fail);
-            return Send(id, true, data);
+            Send(id, true, data);
         }
 
-        public bool Send(Frame frame)
+        public void Send(Frame frame)
         {
             if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Send] Socket status: {Status}");
             var data = NetworkPacketFactory.Message(MessageSerializer.Serialize(frame));
-            return Send(0, false, data);
+            Send(0, false, data);
         }
 
-        public bool Response(byte[] data, int identity)
+        public void Response(byte[] data, int identity)
         {
             if (data == null) throw new ArgumentNullException(nameof(data));
             if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Response] Socket status: {Status}");
-
+            Send(0, false, NetworkPacketFactory.Response(data, identity));
-            return Send(0, false, NetworkPacketFactory.Response(data, identity));
         }
         #endregion
 
@@ -140,7 +154,7 @@ namespace ZeroLevel.Network
             try
             {
                 if (type == FrameType.KeepAlive) return;
-                _incoming_queue.Add(new IncomingFrame
+                _incoming_queue.Enqueue(new IncomingFrame
                 {
                     data = data,
                     type = type,
@@ -223,9 +237,7 @@ namespace ZeroLevel.Network
             IncomingFrame frame = default(IncomingFrame);
             while (Status != SocketClientStatus.Disposed)
             {
-                try
+                if (_incoming_queue.TryDequeue(out frame))
-                {
-                    if (_incoming_queue.TryTake(out frame, 100))
                 {
                     try
                     {
@@ -257,41 +269,62 @@ namespace ZeroLevel.Network
                         Log.SystemError(ex, "[SocketClient.IncomingFramesJob] Handle frame");
                     }
                 }
-                }
+                else
-                catch (Exception ex)
-                {
-                    Log.SystemError(ex, "[SocketClient.IncomingFramesJob] _incoming_queue.Take");
-                    if (Status != SocketClientStatus.Disposed)
                 {
-                        _incoming_queue.Dispose();
+                    Thread.Sleep(100);
-                        _incoming_queue = new BlockingCollection<IncomingFrame>();
-                    }
-                    continue;
                 }
             }
         }
 
-        private bool Send(int id, bool is_request, byte[] data)
+        private void OutcomingFramesJob()
+        {
+            while (Status != SocketClientStatus.Disposed)
             {
                 if (Status == SocketClientStatus.Working)
+                {
+                    if (_outcomingFrameEvent.Wait(100))
+                    {
+                        _outcomingFrameEvent.Reset();
+                    }
+                    while (_outcoming_queue.TryDequeue(out var frame))
                     {
                         try
                         {
-                    if (is_request)
+                            if (frame.is_request)
                             {
-                        _requests.StartSend(id);
+                                _requests.StartSend(frame.identity);
                             }
-                    var sended = _clientSocket.Send(data, data.Length, SocketFlags.None);
+                            _clientSocket.Send(frame.data, frame.data.Length, SocketFlags.None);
-                    return sended == data.Length;
+                            //var sended = _clientSocket.Send(frame.data, frame.data.Length, SocketFlags.None);
+                            //return sended == frame.data.Length;
                         }
                         catch (Exception ex)
                         {
-                    Log.SystemError(ex, $"[SocketClient.SendFramesJob] _str_clientSocketeam.Send");
+                            Log.SystemError(ex, $"[SocketClient.OutcomingFramesJob] _str_clientSocketeam.Send");
                             Broken();
                             OnDisconnect(this);
                         }
+                        finally
+                        {
+                            _outcomingFramesPool.Return(frame);
+                        }
                     }                    
-            return false;
+                }
+                else
+                {
+                    Thread.Sleep(400);
+                }
+            }
+        }
+
+        private void Send(int id, bool is_request, byte[] data)
+        {
+            var frame = _outcomingFramesPool.Get();
+            frame.data = data;
+            frame.identity = id;
+            frame.is_request = is_request;
+            _outcoming_queue.Enqueue(frame);
+            _outcomingFrameEvent.Set();
         }
         #endregion
 

ZeroLevel/Services/Semantic/LanguageDictionary.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using ZeroLevel.Services.Semantic;
 using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel.Services.Semantic

ZeroLevel/Services/Semantic/WordTokenizer.cs

@@ -44,5 +44,41 @@ namespace ZeroLevel.Services.Semantic
                 _pool.Return(buffer);
             }
         }
+
+        public static IEnumerable<string> TokenizeCaseSensitive(string text)
+        {
+            int index = 0;
+            bool first = true;
+            var buffer = _pool.Rent(ARRAY_SIZE);
+            try
+            {
+                for (int i = 0; i < text?.Length; i++)
+                {
+                    if (first && Char.IsLetter(text[i]))
+                    {
+                        first = false;
+                        buffer[index++] = text[i];
+                    }
+                    else if (first == false && Char.IsLetterOrDigit(text[i]))
+                    {
+                        buffer[index++] = text[i];
+                    }
+                    else if (index > 0)
+                    {
+                        yield return new string(buffer, 0, index);
+                        index = 0;
+                        first = true;
+                    }
+                }
+                if (index > 0)
+                {
+                    yield return new string(buffer, 0, index);
+                }
+            }
+            finally
+            {
+                _pool.Return(buffer);
+            }
+        }
     }
 }

ZeroLevel/Services/Text/SuffixAutomata.cs

@@ -0,0 +1,159 @@
+using System.Collections.Generic;
+using System.Linq;
+
+namespace ZeroLevel.Services.Text
+{
+    internal struct state
+    {
+        public int len;
+        public int link;
+        public Dictionary<char, int> next;
+    }
+
+    public class SuffixAutomata
+    {
+        const int MAXLEN = 100000;
+        private state[] st = new state[MAXLEN * 2];
+        int sz, last;
+
+        public void Init(bool is_reused = false)
+        {
+            sz = last = 0;
+            st[0].len = 0;
+            st[0].link = -1;
+            ++sz;
+            // этот код нужен, только если автомат строится много раз для разных строк:
+            for (int i = 0; i < MAXLEN * 2; ++i)
+            {
+                st[i].next = new Dictionary<char, int>();
+            }
+        }
+
+        public void Extend(char c)
+        {
+            int cur = sz++;
+            st[cur].len = st[last].len + 1;
+            int p;
+            for (p = last; p != -1 && !st[p].next.ContainsKey(c); p = st[p].link)
+                st[p].next[c] = cur;
+            if (p == -1)
+                st[cur].link = 0;
+            else
+            {
+                int q = st[p].next[c];
+                if (st[p].len + 1 == st[q].len)
+                    st[cur].link = q;
+                else
+                {
+                    int clone = sz++;
+                    st[clone].len = st[p].len + 1;
+                    st[clone].next = st[q].next;
+                    st[clone].link = st[q].link;
+                    for (; p != -1 && st[p].next.ContainsKey(c) && st[p].next[c] == q; p = st[p].link)
+                        st[p].next[c] = clone;
+                    st[q].link = st[cur].link = clone;
+                }
+            }
+            last = cur;
+        }
+
+        public bool IsSubstring(string w)
+        {
+            if (string.IsNullOrEmpty(w)) return true;
+            bool fail = false;
+            int n, si = 0;
+            for (; si < last; si++)
+            {
+                if (st[si].next.ContainsKey(w[0]))
+                {
+                    var start = st[si];
+                    for (int i = 0; i < w.Length; i++)
+                    {
+                        if (start.next.ContainsKey(w[i]) == false)
+                        {
+                            fail = true;
+                            break;
+                        }
+                        n = start.next[w[i]];
+                        start = st[n];
+                    }
+                    break;
+                }
+            }
+            if (si == last)
+            {
+                fail = true;
+            }
+            return (!fail);
+        }
+
+        public string Intersection(string t)
+        {
+            var entries = st.Where(x => x.next.ContainsKey(t[0])).ToArray();
+            var candidates = entries
+                .Select(s => Intersection(s, t));
+            if (candidates != null && candidates.Any())
+            {
+                var max = candidates.Max(s => s?.Length ?? 0);
+                return candidates.FirstOrDefault(c => c != null && c.Length == max);
+            }
+            return null;
+            /*
+            int v = 0, l = 0, best = 0, bestpos = 0;
+            for (int i = 0; i < (int)t.Length; ++i)
+            {
+                while (v > 0 && !st[v].next.ContainsKey(t[i]))
+                {
+                    v = st[v].link;
+                    l = st[v].len;
+                }
+                if (st[v].next.ContainsKey(t[i]))
+                {
+                    v = st[v].next[t[i]];
+                    ++l;
+                }
+                if (l > best)
+                {
+                    best = l;
+                    bestpos = i;
+                }
+            }
+            var start = bestpos - best + 1;
+            var length = best;
+            if (start >= 0 && start < t.Length && (start + length) <= t.Length)
+                return t.Substring(start, length);
+            return null;
+            */
+        }
+
+        private string Intersection(state entry, string t)
+        {
+            int v = 0, l = 0, best = 0, bestpos = 0;
+            for (int i = 0; i < (int)t.Length; ++i)
+            {
+                while (v > 0 && !entry.next.ContainsKey(t[i]))
+                {
+                    v = entry.link;
+                    l = entry.len;
+                    entry = st[v];
+                }
+                if (entry.next.ContainsKey(t[i]))
+                {
+                    v = entry.next[t[i]];
+                    entry = st[v];
+                    ++l;
+                }
+                if (l > best)
+                {
+                    best = l;
+                    bestpos = i;
+                }
+            }
+            var start = bestpos - best + 1;
+            var length = best;
+            if (start >= 0 && start < t.Length && (start + length) <= t.Length)
+                return t.Substring(start, length);
+            return null;
+        }
+    }
+}

ZeroLevel/Services/Text/TStringReader.cs

@@ -39,9 +39,13 @@
 
         public void SkipBreaks()
         {
-            while (EOF == false && char.IsWhiteSpace(Current)) Move();
+            while (EOF == false && (Current == '\n' || Current == '\r')) Move();
         }
 
+        public bool IsBrackets() => EOF == false && (Current == '\'' || Current == '"' || Current == '”');
+        public bool IsBreak() => EOF == false && (Current == '\n' || Current == '\r');
+        public bool IsWhiteSpace() => EOF == false && char.IsWhiteSpace(Current);
+        
         public bool MoveBack()
         {
             _position = _position - 1;
@@ -84,7 +88,7 @@
                     index++;
                 identity = _template.Substring(offset, index - offset);
             }
-            return identity.ToLowerInvariant();
+            return identity;
         }
 
         public string ReadWord()

ZeroLevel/Services/Transliteration.cs

@@ -22,7 +22,7 @@ namespace ZeroLevel.Services
 
             output = Regex.Replace(output, @"\s|\.|\(", " ");
             output = Regex.Replace(output, @"\s+", " ");
-            output = Regex.Replace(output, @"[^\s\w\d-]", "");
+            //output = Regex.Replace(output, @"[^\s\w\d-]", "");
             output = output.Trim();
 
             Dictionary<string, string> tdict = GetDictionaryByType(type);
@@ -145,7 +145,7 @@ namespace ZeroLevel.Services
             gost.Add("«", "");
             gost.Add("»", "");
             gost.Add("—", "-");
-            gost.Add(" ", "-");
+            gost.Add(" ", " ");
 
             iso.Add("Є", "YE");
             iso.Add("І", "I");
@@ -223,7 +223,7 @@ namespace ZeroLevel.Services
             iso.Add("«", "");
             iso.Add("»", "");
             iso.Add("—", "-");
-            iso.Add(" ", "-");
+            iso.Add(" ", " ");
         }
     }
 }

ZeroLevel/Services/Web/HtmlUtility.cs

@@ -346,7 +346,7 @@ namespace ZeroLevel.Services.Web
                 if (cursor.Current == '&')
                 {
                     cursor.Move();
-                    var identity = cursor.ReadIdentity();
+                    var identity = cursor.ReadIdentity().ToLowerInvariant();
                     cursor.MoveBack();
                     if (_entityMap.ContainsKey(identity + ";"))
                     {

ZeroLevel/ZeroLevel.csproj

@@ -6,7 +6,7 @@
 </Description>
     <Authors>ogoun</Authors>
     <Company>ogoun</Company>
-    <AssemblyVersion>3.3.6.3</AssemblyVersion>
+    <AssemblyVersion>3.3.6.4</AssemblyVersion>
     <PackageReleaseNotes>Configuration floating number convert fix</PackageReleaseNotes>
     <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
     <Copyright>Copyright Ogoun 2022</Copyright>
@@ -14,8 +14,8 @@
     <PackageIconUrl></PackageIconUrl>
     <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
-    <Version>3.3.6.3</Version>
+    <Version>3.3.6.4</Version>
-    <FileVersion>3.3.6.3</FileVersion>
+    <FileVersion>3.3.6.4</FileVersion>
     <Platforms>AnyCPU;x64;x86</Platforms>
     <PackageIcon>zero.png</PackageIcon>
     <DebugType>full</DebugType>