diff --git a/.gitignore b/.gitignore
index 73838a3..27697d7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@ bower_components
*.nupkg
*.p7s
*.sln.ide
+*.onnx
[Bb]in
[Dd]ebug*/
[Oo]bj/
diff --git a/AutoLoader/AppContainer/AppContainer.csproj b/AutoLoader/AppContainer/AppContainer.csproj
new file mode 100644
index 0000000..132c02c
--- /dev/null
+++ b/AutoLoader/AppContainer/AppContainer.csproj
@@ -0,0 +1,9 @@
+
+
+
+ net6.0
+ enable
+ enable
+
+
+
diff --git a/AutoLoader/AppContainer/Class1.cs b/AutoLoader/AppContainer/Class1.cs
new file mode 100644
index 0000000..74f85db
--- /dev/null
+++ b/AutoLoader/AppContainer/Class1.cs
@@ -0,0 +1,50 @@
+namespace AppContainer
+{
+
+ public class Endpoint
+ {
+ public string VersionEndpoint { get; set; }
+ public string AppEndpoint { get; set; }
+ public string PingEndpoint { get; set; }
+ public string Token { get; set; }
+ }
+ public class HostSettings
+ {
+ public IEnumerable Endpoints { get; set; }
+ public string EntryClass { get; set; }
+ public string EntryMethod { get; set; }
+ }
+
+ public class Host
+ {
+ private readonly HostSettings _settings;
+ public Host(HostSettings settings)
+ {
+ if (settings == null) throw new ArgumentNullException(nameof(settings));
+ _settings = settings;
+ }
+
+ public async Task HasUpdate()
+ {
+ if (_settings.Endpoints != null)
+ {
+ foreach (var ep in _settings.Endpoints)
+ {
+ try
+ {
+ using (var client = new HttpClient())
+ {
+ client.DefaultRequestHeaders.Add("token", ep.Token);
+ var response = await client.GetAsync(ep.PingEndpoint);
+ response.EnsureSuccessStatusCode();
+
+ }
+ }
+ catch (Exception ex)
+ {
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/ConnectionTest/Client/Client.csproj b/ConnectionTest/Client/Client.csproj
index 9fbf72d..fc81762 100644
--- a/ConnectionTest/Client/Client.csproj
+++ b/ConnectionTest/Client/Client.csproj
@@ -1,9 +1,10 @@
-
+
Exe
net6.0
- AnyCPU;x64;x86
+ enable
+ enable
diff --git a/ConnectionTest/Client/Program.cs b/ConnectionTest/Client/Program.cs
index 13e8ba7..d8fc73f 100644
--- a/ConnectionTest/Client/Program.cs
+++ b/ConnectionTest/Client/Program.cs
@@ -1,7 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net;
+using System.Net;
using ZeroLevel;
using ZeroLevel.Network;
using ZeroLevel.Services.HashFunctions;
@@ -60,6 +57,9 @@ namespace Client
{
private readonly static XXHashUnsafe _hash = new XXHashUnsafe(667);
+ private static long _successSend = 0;
+ private static long _faultSend = 0;
+
static void Main(string[] args)
{
Log.AddConsoleLogger();
@@ -68,6 +68,12 @@ namespace Client
var port = ReadPort();
ex.RoutesStorage.Set("server", new IPEndPoint(address, port));
+ Sheduller.RemindEvery(TimeSpan.FromMilliseconds(300), () =>
+ {
+ Console.SetCursorPosition(0, 0);
+ Console.WriteLine($"Success/Fault network: {_successSend} / {_faultSend}");
+ });
+
uint index = 0;
while (true)
{
@@ -99,7 +105,7 @@ namespace Client
var info = new Info { Checksum = full_checksum, Id = id, Length = length };
if (client.Request("start", info, res =>
{
- Log.Info($"Success start sending packet '{id}'");
+ Interlocked.Increment(ref _successSend);
}))
{
uint size = 1;
@@ -111,11 +117,15 @@ namespace Client
{
if (!res)
{
- Log.Info($"Fault server incoming packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes.");
+ Interlocked.Increment(ref _faultSend);
+ }
+ else
+ {
+ Interlocked.Increment(ref _successSend);
}
}))
{
- Log.Warning($"Can't start send packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes. No connection");
+ Interlocked.Increment(ref _faultSend);
}
offset += size;
size += 1;
@@ -124,7 +134,7 @@ namespace Client
}
else
{
- Log.Warning($"Can't start send packet '{id}'. No connection");
+ Interlocked.Increment(ref _faultSend);
}
}
@@ -135,7 +145,7 @@ namespace Client
var info = new Info { Checksum = full_checksum, Id = id, Length = length };
if (exchange.Request("server", "start", info))
{
- Log.Info($"Success start sending packet '{id}'");
+ Interlocked.Increment(ref _successSend);
uint size = 4096;
uint offset = 0;
while (offset < payload.Length)
@@ -143,7 +153,11 @@ namespace Client
var fragment = GetFragment(id, payload, offset, size);
if (!exchange.Request("server", "part", fragment))
{
- Log.Info($"Fault server incoming packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes.");
+ Interlocked.Increment(ref _faultSend);
+ }
+ else
+ {
+ Interlocked.Increment(ref _successSend);
}
offset += size;
}
@@ -151,7 +165,7 @@ namespace Client
}
else
{
- Log.Warning($"Can't start send packet '{id}'. No connection");
+ Interlocked.Increment(ref _faultSend);
}
}
diff --git a/ConnectionTest/Client/Properties/PublishProfiles/FolderProfile.pubxml b/ConnectionTest/Client/Properties/PublishProfiles/FolderProfile.pubxml
deleted file mode 100644
index ccb19b1..0000000
--- a/ConnectionTest/Client/Properties/PublishProfiles/FolderProfile.pubxml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
- FileSystem
- Release
- Any CPU
- netcoreapp3.1
- bin\Release\netcoreapp3.1\publish\
-
-
\ No newline at end of file
diff --git a/ConnectionTest/Server/Program.cs b/ConnectionTest/Server/Program.cs
index 40701f1..75657cc 100644
--- a/ConnectionTest/Server/Program.cs
+++ b/ConnectionTest/Server/Program.cs
@@ -6,6 +6,7 @@ using ZeroLevel.Services.Serialization;
namespace Server
{
+ //netsh advfirewall firewall add rule name="ClientTest" dir=in action=allow protocol=TCP localport=5016
public class Info
: IBinarySerializable
{
diff --git a/TestApp/HybrydLock.cs b/TestApp/HybrydLock.cs
deleted file mode 100644
index abc7bc9..0000000
--- a/TestApp/HybrydLock.cs
+++ /dev/null
@@ -1,28 +0,0 @@
-using System.Threading;
-
-namespace TestApp
-{
- public class HybrydLock
- {
- private AutoResetEvent _lock = new AutoResetEvent(false);
- private volatile int _counter = 0;
-
- public void Enter()
- {
- if (Interlocked.Increment(ref _counter) == 1)
- {
- return;
- }
- _lock.WaitOne();
- }
-
- public void Leave()
- {
- if (Interlocked.Decrement(ref _counter) == 0)
- {
- return;
- }
- _lock.Set();
- }
- }
-}
diff --git a/TestApp/MyService.cs b/TestApp/MyService.cs
deleted file mode 100644
index 213cf4e..0000000
--- a/TestApp/MyService.cs
+++ /dev/null
@@ -1,72 +0,0 @@
-using System;
-using System.Net;
-using System.Threading;
-using ZeroLevel;
-using ZeroLevel.Network;
-using ZeroLevel.Services.Applications;
-
-namespace TestApp
-{
- public class MyService
- : BaseZeroService
- {
- public MyService()
- : base()
- {
- }
-
- protected override void StartAction()
- {
- Log.Info("Started");
- ReadServiceInfo();
- var host = UseHost(8800);
- AutoregisterInboxes(host);
- host.OnConnect += Host_OnConnect;
- host.OnDisconnect += Host_OnDisconnect;
-
- int counter = 0;
- Sheduller.RemindEvery(TimeSpan.FromSeconds(1), () =>
- {
- Log.Info($"RPS: {counter}");
- Interlocked.Exchange(ref counter, 0);
- });
-
- Exchange.RoutesStorage.Set("test.app", new IPEndPoint(IPAddress.Loopback, 8800));
-
- while (true)
- {
- try
- {
- //var s = Exchange.Request("test.app", "counter");
- Interlocked.Add(ref counter, Exchange.Request("test.app", "counter"));
- }
- catch(Exception ex)
- {
- Log.Error(ex, "Request fault");
- Thread.Sleep(300);
- }
- }
- }
-
- private void Host_OnDisconnect(ISocketClient obj)
- {
- Log.Info($"Client '{obj.Endpoint.Address}:{obj.Endpoint.Port}' disconnected");
- }
-
- private void Host_OnConnect(IClient obj)
- {
- Log.Info($"Client '{obj.Socket.Endpoint.Address}:{obj.Socket.Endpoint.Port}' connected");
- }
-
- [ExchangeReplierWithoutArg("counter")]
- public int GetCounter(ISocketClient client)
- {
- return 1;
- }
-
- protected override void StopAction()
- {
- Log.Info("Stopped");
- }
- }
-}
\ No newline at end of file
diff --git a/TestApp/PersonDetector.cs b/TestApp/PersonDetector.cs
new file mode 100644
index 0000000..a3415c8
--- /dev/null
+++ b/TestApp/PersonDetector.cs
@@ -0,0 +1,44 @@
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.PixelFormats;
+using System.Collections.Generic;
+using System.Linq;
+using ZeroLevel.NN.Architectures.YoloV5;
+using ZeroLevel.NN.Models;
+using ZeroLevel.NN.Services;
+
+namespace TestApp
+{
+ public class PersonDetector
+ {
+ private const string MODEL_PATH = @"nnmodels/Yolo5S/yolov5s327e.onnx";
+ private readonly Yolov5Detector _detector;
+ private float _threshold = 0.17f;
+ public PersonDetector()
+ {
+ _detector = new Yolov5Detector(MODEL_PATH, gpu: false);
+ }
+
+ public IEnumerable Detect(string imagePath)
+ {
+ using (Image image = Image.Load(imagePath))
+ {
+ var t_predictions = _detector.PredictMultiply(image, true, _threshold);
+ t_predictions.Apply(p =>
+ {
+ p.Cx /= image.Width;
+ p.Cy /= image.Height;
+ });
+ if (t_predictions != null)
+ {
+ t_predictions.RemoveAll(p => (p.W * image.Width) < 10.0f || (p.H * image.Height) < 10.0f);
+ }
+ if (t_predictions.Count > 0)
+ {
+ NMS.Apply(t_predictions);
+ return t_predictions;
+ }
+ }
+ return Enumerable.Empty();
+ }
+ }
+}
diff --git a/TestApp/Program.cs b/TestApp/Program.cs
index 593761c..b4b4a9b 100644
--- a/TestApp/Program.cs
+++ b/TestApp/Program.cs
@@ -1,84 +1,11 @@
-using Newtonsoft.Json;
-using System;
-using System.Net;
-using System.Threading;
-using ZeroLevel;
-using ZeroLevel.Logging;
-using ZeroLevel.Network;
-using ZeroLevel.Services.Serialization;
-using ZeroLevel.Services.Trees;
-
-namespace TestApp
+namespace TestApp
{
- public class TestQuery
- {
- public string Name { get; set; }
- public int Age { get; set; }
- public string[] Roles { get; set; }
- }
-
internal static class Program
{
- private static string Serialize(object instance)
- {
- return JsonConvert.SerializeObject(instance);
- }
-
private static void Main(string[] args)
{
- Configuration.Save(Configuration.ReadFromApplicationConfig());
- Bootstrap.Startup(args,
- () => Configuration.ReadSetFromIniFile("config.ini"))
- .EnableConsoleLog(LogLevel.System | LogLevel.FullDebug)
- //.UseDiscovery()
- .Run()
- .WaitWhileStatus(ZeroServiceStatus.Running)
- .Stop();
- Bootstrap.Shutdown();
- }
-
- static void SimpleCSTest()
- {
- var server_router = new Router();
- server_router.RegisterInbox("test", (c, line) =>
- {
- Console.WriteLine(line);
- });
-
- server_router.RegisterInbox("req", (c, line) =>
- {
- Console.WriteLine($"Request: {line}");
- return line.ToUpperInvariant();
- });
-
- var server = new SocketServer(new System.Net.IPEndPoint(IPAddress.Any, 666), server_router);
-
-
- var client_router = new Router();
- var client = new SocketClient(new IPEndPoint(IPAddress.Loopback, 666), client_router);
-
- var frm = FrameFactory.Create("req", MessageSerializer.SerializeCompatible("Hello world"));
-
- while (Console.KeyAvailable == false)
- {
- client.Request(frm, data =>
- {
- var line = MessageSerializer.DeserializeCompatible(data);
- Console.WriteLine($"Response: {line}");
- });
- Thread.Sleep(2000);
- }
- }
-
- public static double[] Generate(int vector_size)
- {
- var rnd = new Random((int)Environment.TickCount);
- var vector = new double[vector_size];
- for (int i = 0; i < vector_size; i++)
- {
- vector[i] = 50.0d - rnd.NextDouble() * 100.0d;
- }
- return vector;
+ var detector = new PersonDetector();
+ var predictions = detector.Detect(@"E:\Desktop\test\1.JPG");
}
}
}
\ No newline at end of file
diff --git a/TestApp/TestApp.csproj b/TestApp/TestApp.csproj
index 844a5fb..480e7bd 100644
--- a/TestApp/TestApp.csproj
+++ b/TestApp/TestApp.csproj
@@ -11,6 +11,7 @@
+
@@ -18,6 +19,9 @@
Always
+
+ PreserveNewest
+
diff --git a/ZeroLevel.NN/Architectures/AgeDetectors/GenderAgeEstimator.cs b/ZeroLevel.NN/Architectures/AgeDetectors/GenderAgeEstimator.cs
new file mode 100644
index 0000000..8315304
--- /dev/null
+++ b/ZeroLevel.NN/Architectures/AgeDetectors/GenderAgeEstimator.cs
@@ -0,0 +1,37 @@
+using Microsoft.ML.OnnxRuntime.Tensors;
+using SixLabors.ImageSharp;
+using ZeroLevel.NN.Models;
+
+namespace ZeroLevel.NN
+{
+ public class GenderAgeEstimator
+ : SSDNN
+ {
+ private const int INPUT_WIDTH = 64;
+ private const int INPUT_HEIGHT = 64;
+
+ public GenderAgeEstimator(string modelPath, bool gpu = false) : base(modelPath, gpu)
+ {
+ }
+
+ public (Gender, int) Predict(Image image)
+ {
+ var input = MakeInput(image,
+ new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst)
+ .ApplyNormilization()
+ .ApplyAxeInversion());
+ return Predict(input);
+ }
+
+ public (Gender, int) Predict(Tensor input)
+ {
+ float[] variances = null;
+ Extract(new Dictionary> { { "input", input } }, d =>
+ {
+ variances = d.First().Value.ToArray();
+ });
+ var gender = Argmax(variances[0..2]) == 0 ? Gender.Male : Gender.Feemale;
+ return (gender, (int)variances[2]);
+ }
+ }
+}
diff --git a/ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeDetector.cs b/ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeEstimator.cs
similarity index 88%
rename from ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeDetector.cs
rename to ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeEstimator.cs
index 04f0d38..53f3a6d 100644
--- a/ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeDetector.cs
+++ b/ZeroLevel.NN/Architectures/AgeDetectors/GoogleAgeEstimator.cs
@@ -19,7 +19,7 @@ namespace ZeroLevel.NN
///
/// Input tensor is 1 x 3 x height x width with mean values 104, 117, 123. Input image have to be previously resized to 224 x 224 pixels and converted to BGR format.
///
- public class GoogleAgeDetector
+ public class GoogleAgeEstimator
: SSDNN
{
private const int INPUT_WIDTH = 224;
@@ -28,7 +28,7 @@ namespace ZeroLevel.NN
private Age[] _ageList = new[] { Age.From0To2, Age.From4To6, Age.From8To12, Age.From15To20, Age.From25To32, Age.From38To43, Age.From48To53, Age.From60To100 };
- public GoogleAgeDetector(string modelPath, bool gpu = false) : base(modelPath, gpu)
+ public GoogleAgeEstimator(string modelPath, bool gpu = false) : base(modelPath, gpu)
{
}
@@ -48,7 +48,7 @@ namespace ZeroLevel.NN
{
variances = d.First().Value.ToArray();
});
- var (number, index) = variances.Select((n, i) => (n, i)).Max();
+ var index = Argmax(variances);
return _ageList[index];
}
}
diff --git a/ZeroLevel.NN/Architectures/FaceSeacrhService.cs b/ZeroLevel.NN/Architectures/FaceSeacrhService.cs
index 68f79d9..68543a7 100644
--- a/ZeroLevel.NN/Architectures/FaceSeacrhService.cs
+++ b/ZeroLevel.NN/Architectures/FaceSeacrhService.cs
@@ -1,6 +1,7 @@
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
+using ZeroLevel;
using ZeroLevel.NN;
using ZeroLevel.NN.Models;
@@ -127,25 +128,36 @@ namespace Zero.NN.Services
var around_x2 = centerFaceX + radius;
var around_y1 = centerFaceY - radius;
var around_y2 = centerFaceY + radius;
- using (var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2))
+ try
{
- var matrix = Face.GetTransformMatrix(face);
- var builder = new AffineTransformBuilder();
- builder.AppendMatrix(matrix);
- faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic));
- vector = _encoder.Predict(faceImage);
- /*var aligned_faces = detector.Predict(faceImage);
- if (aligned_faces != null && aligned_faces.Count == 1)
+ using (var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2))
{
- using (var ci = SpecialCrop(faceImage, aligned_faces[0]))
+ var matrix = Face.GetTransformMatrix(face);
+ var builder = new AffineTransformBuilder();
+ builder.AppendMatrix(matrix);
+ faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic));
+ vector = _encoder.Predict(faceImage);
+ /*var aligned_faces = detector.Predict(faceImage);
+ if (aligned_faces != null && aligned_faces.Count == 1)
{
- vector = encoder.Predict(faceImage);
+ using (var ci = SpecialCrop(faceImage, aligned_faces[0]))
+ {
+ vector = encoder.Predict(faceImage);
+ }
}
+ else
+ {
+ vector = encoder.Predict(faceImage);
+ }*/
}
- else
+ }
+ catch (Exception ex)
+ {
+ Log.SystemError(ex, "[FaceSeacrhService.GetEmbeddings]");
+ using (var faceImage = ImagePreprocessor.Crop(image, face.X1, face.Y1, face.X2, face.Y2))
{
- vector = encoder.Predict(faceImage);
- }*/
+ vector = _encoder.Predict(faceImage);
+ }
}
}
else
@@ -162,5 +174,53 @@ namespace Zero.NN.Services
};
}
}
+
+
+ public IEnumerable<(FaceEmbedding, Image)> GetEmbeddingsAndCrop(Image image)
+ {
+ int width = image.Width;
+ int heigth = image.Height;
+ var faces = _detector.Predict(image);
+ foreach (var face in faces)
+ {
+ Face.FixInScreen(face, width, heigth);
+ float[] vector;
+ if (_useFaceAlign)
+ {
+ int x = (int)face.X1;
+ int y = (int)face.Y1;
+ int w = (int)(face.X2 - face.X1);
+ int h = (int)(face.Y2 - face.Y1);
+ var radius = (float)Math.Sqrt(w * w + h * h) / 2f;
+ var centerFaceX = (face.X2 + face.X1) / 2.0f;
+ var centerFaceY = (face.Y2 + face.Y1) / 2.0f;
+ var around_x1 = centerFaceX - radius;
+ var around_x2 = centerFaceX + radius;
+ var around_y1 = centerFaceY - radius;
+ var around_y2 = centerFaceY + radius;
+ var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2);
+ var matrix = Face.GetTransformMatrix(face);
+ var builder = new AffineTransformBuilder();
+ builder.AppendMatrix(matrix);
+ faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic));
+ vector = _encoder.Predict(faceImage);
+ yield return (new FaceEmbedding
+ {
+ Face = face,
+ Vector = vector
+ }, faceImage);
+ }
+ else
+ {
+ var faceImage = ImagePreprocessor.Crop(image, face.X1, face.Y1, face.X2, face.Y2);
+ vector = _encoder.Predict(faceImage);
+ yield return (new FaceEmbedding
+ {
+ Face = face,
+ Vector = vector
+ }, faceImage);
+ }
+ }
+ }
}
}
diff --git a/ZeroLevel.NN/Architectures/YoloV5/Yolov5Detector.cs b/ZeroLevel.NN/Architectures/YoloV5/Yolov5Detector.cs
new file mode 100644
index 0000000..855dc43
--- /dev/null
+++ b/ZeroLevel.NN/Architectures/YoloV5/Yolov5Detector.cs
@@ -0,0 +1,144 @@
+using Microsoft.ML.OnnxRuntime.Tensors;
+using SixLabors.ImageSharp;
+using ZeroLevel.NN.Models;
+
+namespace ZeroLevel.NN.Architectures.YoloV5
+{
+ public class Yolov5Detector
+ : SSDNN
+ {
+ private int INPUT_WIDTH = 640;
+ private int INPUT_HEIGHT = 640;
+ private int CROP_WIDTH = 1440;
+ private int CROP_HEIGHT = 1440;
+
+ public Yolov5Detector(string modelPath, int inputWidth = 640, int inputHeight = 640, bool gpu = false)
+ : base(modelPath, gpu)
+ {
+ INPUT_HEIGHT = inputHeight;
+ INPUT_WIDTH = inputWidth;
+ }
+
+ public List Predict(Image image, float threshold)
+ {
+ var input = MakeInput(image,
+ new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst)
+ .ApplyNormilization()
+ .ApplyAxeInversion());
+ return Predict(input, threshold);
+ }
+
+ public List PredictMultiply(Image image, bool withFullResizedImage, float threshold)
+ {
+ var input = MakeInputBatch(image,
+ new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst)
+ .ApplyNormilization()
+ .ApplyAxeInversion()
+ .UseCrop(CROP_WIDTH, CROP_HEIGHT, withFullResizedImage, true));
+ return PredictMultiply(input, threshold);
+ }
+
+ public List Predict(Tensor input, float threshold)
+ {
+ var result = new List();
+ Extract(new Dictionary> { { "images", input } }, d =>
+ {
+ var output = d["output"];
+ /*
+ var output350 = d["350"];
+ var output498 = d["498"];
+ var output646 = d["646"];
+ */
+ if (output != null && output != null)
+ {
+ var relative_koef_x = 1.0f / INPUT_WIDTH;
+ var relative_koef_y = 1.0f / INPUT_HEIGHT;
+ for (int box = 0; box < output.Dimensions[1]; box++)
+ {
+ var conf = output[0, box, 4]; // уверенность в наличии любого объекта
+ if (conf > threshold)
+ {
+ var class_confidense = output[0, box, 5]; // уверенность в наличии объекта класса person
+ if (class_confidense > threshold)
+ {
+ // Перевод относительно входа модели в относительные координаты
+ var cx = output[0, box, 0] * relative_koef_x;
+ var cy = output[0, box, 1] * relative_koef_y;
+ var h = output[0, box, 2] * relative_koef_y;
+ var w = output[0, box, 3] * relative_koef_x;
+ result.Add(new YoloPrediction
+ {
+ Cx = cx,
+ Cy = cy,
+ W = w,
+ H = h,
+ Class = 0,
+ Label = "0",
+ Score = conf
+ });
+ }
+ }
+ }
+ }
+ });
+ return result;
+ }
+
+ public List PredictMultiply(ImagePredictionInput[] inputs, float threshold)
+ {
+ var result = new List();
+ var relative_koef_x = 1.0f / (float)INPUT_WIDTH;
+ var relative_koef_y = 1.0f / (float)INPUT_HEIGHT;
+ foreach (var input in inputs)
+ {
+ Extract(new Dictionary> { { "images", input.Tensor } }, d =>
+ {
+ var output = d["output"];
+ /*
+ var output350 = d["350"];
+ var output498 = d["498"];
+ var output646 = d["646"];
+ */
+ if (output != null && output != null)
+ {
+ for (int index = 0; index < input.Count; index++)
+ {
+ var real_koef_x = (float)input.Offsets[index].Width / (float)INPUT_WIDTH;
+ var real_koef_y = (float)input.Offsets[index].Height / (float)INPUT_HEIGHT;
+ for (int box = 0; box < output.Dimensions[1]; box++)
+ {
+ var conf = output[index, box, 4]; // уверенность в наличии любого объекта
+ if (conf > threshold)
+ {
+ var class_confidense = output[index, box, 5]; // уверенность в наличии объекта класса person
+ if (class_confidense > threshold)
+ {
+ // Перевод относительно входа модели в относительные координаты
+ var cx = output[index, box, 0] * real_koef_x;
+ var cy = output[index, box, 1] * real_koef_y;
+ var h = output[index, box, 2] * relative_koef_y;
+ var w = output[index, box, 3] * relative_koef_x;
+ // Перевод в координаты отнисительно текущего смещения
+ cx += input.Offsets[index].X;
+ cy += input.Offsets[index].Y;
+ result.Add(new YoloPrediction
+ {
+ Cx = cx,
+ Cy = cy,
+ W = w,
+ H = h,
+ Class = 0,
+ Label = "0",
+ Score = conf
+ });
+ }
+ }
+ }
+ }
+ }
+ });
+ }
+ return result;
+ }
+ }
+}
diff --git a/ZeroLevel.NN/Models/Gender.cs b/ZeroLevel.NN/Models/Gender.cs
new file mode 100644
index 0000000..8fc5795
--- /dev/null
+++ b/ZeroLevel.NN/Models/Gender.cs
@@ -0,0 +1,9 @@
+namespace ZeroLevel.NN.Models
+{
+ public enum Gender
+ {
+ Unknown = 0,
+ Male = 1,
+ Feemale = 2
+ }
+}
diff --git a/ZeroLevel.NN/Models/PredictionInput.cs b/ZeroLevel.NN/Models/ImagePredictionInput.cs
similarity index 83%
rename from ZeroLevel.NN/Models/PredictionInput.cs
rename to ZeroLevel.NN/Models/ImagePredictionInput.cs
index 438f42f..3934037 100644
--- a/ZeroLevel.NN/Models/PredictionInput.cs
+++ b/ZeroLevel.NN/Models/ImagePredictionInput.cs
@@ -2,7 +2,7 @@
namespace ZeroLevel.NN.Models
{
- public class PredictionInput
+ public class ImagePredictionInput
{
public Tensor Tensor;
public OffsetBox[] Offsets;
diff --git a/ZeroLevel.NN/Models/ImagePreprocessorOptions.cs b/ZeroLevel.NN/Models/ImagePreprocessorOptions.cs
index 2566676..91e783f 100644
--- a/ZeroLevel.NN/Models/ImagePreprocessorOptions.cs
+++ b/ZeroLevel.NN/Models/ImagePreprocessorOptions.cs
@@ -2,7 +2,7 @@
{
public class ImagePreprocessorOptions
{
- private const float PIXEL_NORMALIZATION_SCALE = 1f / 255f;
+ private const float PIXEL_NORMALIZATION_SCALE = 1.0f / 255.0f;
public ImagePreprocessorOptions(int inputWidth, int inputHeight, PredictorChannelType channelType)
{
this.InputWidth = inputWidth;
diff --git a/ZeroLevel.NN/Models/YoloPrediction.cs b/ZeroLevel.NN/Models/YoloPrediction.cs
new file mode 100644
index 0000000..2ae7f87
--- /dev/null
+++ b/ZeroLevel.NN/Models/YoloPrediction.cs
@@ -0,0 +1,66 @@
+using ZeroLevel.Services.Serialization;
+
+namespace ZeroLevel.NN.Models
+{
+ public class YoloPrediction
+ : IBinarySerializable
+ {
+ public int Class { get; set; }
+ public float Cx { get; set; }
+ public float Cy { get; set; }
+ public float W { get; set; }
+ public float H { get; set; }
+ public float Score { get; set; }
+ public string Label { get; set; }
+
+ public float X { get { return Cx - W / 2.0f; } }
+ public float Y { get { return Cy - W / 2.0f; } }
+
+ public float Area { get { return W * H; } }
+
+ public string Description
+ {
+ get
+ {
+ return $"{Label} ({(int)(Score * 100)} %)";
+ }
+ }
+
+ public float this[int index]
+ {
+ get
+ {
+ switch (index)
+ {
+ case 0: return Cx;
+ case 1: return Cy;
+ case 2: return Cx + W;
+ case 3: return Cy + H;
+ }
+ return 0;
+ }
+ }
+
+ public void Deserialize(IBinaryReader reader)
+ {
+ this.Cx = reader.ReadFloat();
+ this.Cy = reader.ReadFloat();
+ this.W = reader.ReadFloat();
+ this.H = reader.ReadFloat();
+ this.Class = reader.ReadInt32();
+ this.Score = reader.ReadFloat();
+ this.Label = reader.ReadString();
+ }
+
+ public void Serialize(IBinaryWriter writer)
+ {
+ writer.WriteFloat(this.Cx);
+ writer.WriteFloat(this.Cy);
+ writer.WriteFloat(this.W);
+ writer.WriteFloat(this.H);
+ writer.WriteInt32(this.Class);
+ writer.WriteFloat(this.Score);
+ writer.WriteString(this.Label);
+ }
+ }
+}
diff --git a/ZeroLevel.NN/Services/ImagePreprocessor.cs b/ZeroLevel.NN/Services/ImagePreprocessor.cs
index b5e824e..4a07855 100644
--- a/ZeroLevel.NN/Services/ImagePreprocessor.cs
+++ b/ZeroLevel.NN/Services/ImagePreprocessor.cs
@@ -8,6 +8,8 @@ namespace ZeroLevel.NN
{
public static class ImagePreprocessor
{
+ private static Action, float, int, int, int, int> _precompiledChannelFirstAction = new Action, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, c, i, j] = v; });
+ private static Action, float, int, int, int, int> _precompiledChannelLastAction = new Action, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, i, j, c] = v; });
private static Func PixelToTensorMethod(ImagePreprocessorOptions options)
{
if (options.Normalize)
@@ -42,26 +44,51 @@ namespace ZeroLevel.NN
return new Func((b, _) => (float)b);
}
+ //private static int CalculateFragmentsCount(Image image, ImagePreprocessorOptions options)
+ //{
+ // int count = (options.Crop.SaveOriginal ? 1 : 0);
+ // var Sw = image.Width; // ширина оригинала
+ // var Sh = image.Height; // высота оригинала
+
+ // var CRw = options.Crop.Width; // ширина кропа
+ // var CRh = options.Crop.Height; // высота кропа
+
+ // var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw; // сдвиг по OX к следующему кропу
+ // var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh; // сдвиг по OY к следующему кропу
+
+ // for (int x = 0; x < Sw; x += Dx)
+ // {
+ // for (int y = 0; y < Sh; y += Dy)
+ // {
+ // count++;
+ // }
+ // }
+ // return count;
+ //}
private static int CalculateFragmentsCount(Image image, ImagePreprocessorOptions options)
{
- int count = 0;
- var xs = options.Crop.Overlap ? (int)(options.Crop.Width * options.Crop.OverlapKoefWidth) : options.Crop.Width;
- var ys = options.Crop.Overlap ? (int)(options.Crop.Height * options.Crop.OverlapKoefHeight) : options.Crop.Height;
- for (var x = 0; x < image.Width - xs; x += xs)
+ int count = (options.Crop.SaveOriginal ? 1 : 0);
+ var Sw = image.Width; // ширина оригинала
+ var Sh = image.Height; // высота оригинала
+
+ var CRw = options.InputWidth; // ширина кропа (равна ширине входа, т.к. изображение отресайзено подобающим образом)
+ var CRh = options.InputHeight; // высота кропа (равна высоте входа, т.к. изображение отресайзено подобающим образом)
+
+ var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw; // сдвиг по OX к следующему кропу
+ var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh; // сдвиг по OY к следующему кропу
+
+ for (int x = 0; x < Sw; x += Dx)
{
- for (var y = 0; y < image.Height - ys; y += ys)
+ for (int y = 0; y < Sh; y += Dy)
{
count++;
}
}
return count;
}
-
private static void FillTensor(Tensor tensor, Image image, int index, ImagePreprocessorOptions options, Func pixToTensor)
{
- var append = options.ChannelType == PredictorChannelType.ChannelFirst
- ? new Action, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, c, i, j] = v; })
- : new Action, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, i, j, c] = v; });
+ var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction;
((Image)image).ProcessPixelRows(pixels =>
{
@@ -112,6 +139,59 @@ namespace ZeroLevel.NN
});
}
+ private static void FillTensor(Tensor tensor, Image image, int startX, int startY, int w, int h, int index, ImagePreprocessorOptions options, Func pixToTensor)
+ {
+ var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction;
+
+ ((Image)image).ProcessPixelRows(pixels =>
+ {
+ if (options.InvertXY)
+ {
+ for (int y = startY; y < h; y++)
+ {
+ Span pixelSpan = pixels.GetRowSpan(y);
+ for (int x = startX; x < w; x++)
+ {
+ if (options.BGR)
+ {
+ append(tensor, pixToTensor(pixelSpan[x].B, 0), index, 0, y, x);
+ append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, y, x);
+ append(tensor, pixToTensor(pixelSpan[x].R, 2), index, 2, y, x);
+ }
+ else
+ {
+ append(tensor, pixToTensor(pixelSpan[x].R, 0), index, 0, y, x);
+ append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, y, x);
+ append(tensor, pixToTensor(pixelSpan[x].B, 2), index, 2, y, x);
+ }
+ }
+ }
+ }
+ else
+ {
+ for (int y = startY; y < h; y++)
+ {
+ Span pixelSpan = pixels.GetRowSpan(y);
+ for (int x = startX; x < w; x++)
+ {
+ if (options.BGR)
+ {
+ append(tensor, pixToTensor(pixelSpan[x].B, 0), index, 0, x, y);
+ append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, x, y);
+ append(tensor, pixToTensor(pixelSpan[x].R, 2), index, 2, x, y);
+ }
+ else
+ {
+ append(tensor, pixToTensor(pixelSpan[x].R, 0), index, 0, x, y);
+ append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, x, y);
+ append(tensor, pixToTensor(pixelSpan[x].B, 2), index, 2, x, y);
+ }
+ }
+ }
+ }
+ });
+ }
+
private static Tensor InitInputTensor(ImagePreprocessorOptions options, int batchSize = 1)
{
switch (options.ChannelType)
@@ -127,101 +207,139 @@ namespace ZeroLevel.NN
}
}
- public static PredictionInput[] ToTensors(this Image image, ImagePreprocessorOptions options)
+
+ public static ImagePredictionInput[] ToTensors(this Image image, ImagePreprocessorOptions options)
{
- PredictionInput[] result = null;
+ ImagePredictionInput[] result = null;
var pixToTensor = PixelToTensorMethod(options);
options.Channels = image.PixelType.BitsPerPixel >> 3;
-
if (options.Crop.Enabled)
{
- var fragments = CalculateFragmentsCount(image, options);
- int count = CalculateFragmentsCount(image, options) + (options.Crop.SaveOriginal ? 1 : 0);
- int offset = count % options.MaxBatchSize;
- int count_tensors = count / options.MaxBatchSize + (offset == 0 ? 0 : 1);
- var tensors = new PredictionInput[count_tensors];
- for (int i = 0; i < count_tensors; i++)
- {
- if (i < count_tensors - 1)
- {
- tensors[i] = new PredictionInput
- {
- Tensor = InitInputTensor(options, options.MaxBatchSize),
- Offsets = new OffsetBox[options.MaxBatchSize],
- Count = options.MaxBatchSize
- };
- }
- else
- {
- tensors[i] = new PredictionInput
- {
- Tensor = InitInputTensor(options, offset == 0 ? options.MaxBatchSize : offset),
- Offsets = new OffsetBox[offset == 0 ? options.MaxBatchSize : offset],
- Count = offset == 0 ? options.MaxBatchSize : offset
- };
- }
- }
+ // Размеры оригинального изображения
+ var Sw = image.Width;
+ var Sh = image.Height;
- int tensor_index = 0;
- int tensor_part_index = 0;
- var xs = options.Crop.Overlap ? (int)(options.Crop.Width * options.Crop.OverlapKoefWidth) : options.Crop.Width;
- var ys = options.Crop.Overlap ? (int)(options.Crop.Height * options.Crop.OverlapKoefHeight) : options.Crop.Height;
+ // Создание ресайза для целочисленного прохода кропами шириной CRw и высотой CRh
+ var resizedForCropWidthKoef = options.InputWidth / (double)options.Crop.Width;
+ var resizedForCropHeightKoef = options.InputHeight / (double)options.Crop.Height;
- if (options.Crop.SaveOriginal)
- {
- using (var copy = image.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
- {
- FillTensor(tensors[tensor_index].Tensor, copy, tensor_part_index, options, pixToTensor);
- tensors[tensor_index].Offsets[tensor_part_index] = new OffsetBox(0, 0, image.Width, image.Height);
- }
- tensor_part_index++;
- }
- for (var x = 0; x < image.Width - xs; x += xs)
+ // Размеры для ресайза изображения к размеру по которому удобно идти кропами
+ var resizedForCropWidth = (int)Math.Round(Sw * resizedForCropWidthKoef, MidpointRounding.ToEven);
+ var resizedForCropHeight = (int)Math.Round(Sh * resizedForCropHeightKoef, MidpointRounding.ToEven);
+
+ // Размеры кропа, равны входу сети, а не (options.Crop.Width, options.Crop.Height), т.к. для оптимизации изображение будет предварительно отресайзено
+ var CRw = options.InputWidth;
+ var CRh = options.InputHeight;
+
+ // Расчет сдвигов между кропами
+ var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw;
+ var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh;
+
+ using (var source = image.Clone(img => img.Resize(resizedForCropWidth, resizedForCropHeight, KnownResamplers.Bicubic)))
{
- var startx = x;
- var dx = (x + options.Crop.Width) - image.Width;
- if (dx > 0)
- {
- startx -= dx;
- }
- for (var y = 0; y < image.Height - ys; y += ys)
+ // Количество тензоров всего, во всех батчах суммарно
+ var count = CalculateFragmentsCount(source, options);
+
+ // Проверка, укладывается ли количество тензоров поровну в батчи
+ int offset = count % options.MaxBatchSize;
+
+ // Количество батчей
+ int count_tensor_batches = count / options.MaxBatchSize + (offset == 0 ? 0 : 1);
+
+ // Батчи
+ var tensors = new ImagePredictionInput[count_tensor_batches];
+
+ // Инициализация батчей
+ Parallel.For(0, count_tensor_batches, batch_index =>
{
- if (tensor_part_index > 0 && tensor_part_index % options.MaxBatchSize == 0)
+ if (batch_index < count_tensor_batches - 1)
{
- tensor_index++;
- tensor_part_index = 0;
+ tensors[batch_index] = new ImagePredictionInput
+ {
+ Tensor = InitInputTensor(options, options.MaxBatchSize),
+ Offsets = new OffsetBox[options.MaxBatchSize],
+ Count = options.MaxBatchSize
+ };
}
- var starty = y;
- var dy = (y + options.Crop.Height) - image.Height;
- if (dy > 0)
+ else
{
- starty -= dy;
+ tensors[batch_index] = new ImagePredictionInput
+ {
+ Tensor = InitInputTensor(options, offset == 0 ? options.MaxBatchSize : offset),
+ Offsets = new OffsetBox[offset == 0 ? options.MaxBatchSize : offset],
+ Count = offset == 0 ? options.MaxBatchSize : offset
+ };
}
- using (var copy = image
- .Clone(img => img
- .Crop(new Rectangle(startx, starty, options.Crop.Width, options.Crop.Height))
- .Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
+ });
+
+ // Заполнение батчей
+ int tensor_index = 0;
+
+ // Если используется ресайз оригинала кроме кропов, пишется в первый батч в первый тензор
+ if (options.Crop.SaveOriginal)
+ {
+ using (var copy = source.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
{
- FillTensor(tensors[tensor_index].Tensor, copy, tensor_part_index, options, pixToTensor);
- tensors[tensor_index].Offsets[tensor_part_index] = new OffsetBox(startx, starty, options.Crop.Width, options.Crop.Height);
+ FillTensor(tensors[0].Tensor, copy, 0, options, pixToTensor);
+ tensors[tensor_index].Offsets[0] = new OffsetBox(0, 0, image.Width, image.Height);
}
- tensor_part_index++;
+ tensor_index++;
}
+ tensor_index--;
+ Parallel.ForEach(SteppedIterator(0, source.Width, Dx), x =>
+ {
+ // Можно запараллелить и тут, но выигрыш дает малоощутимый
+ for (int y = 0; y < source.Height; y += Dy)
+ {
+ var current_index = Interlocked.Increment(ref tensor_index);
+ // Индекс тензора внутри батча
+ var b_index = current_index % options.MaxBatchSize;
+ // Индекс батча
+ var p_index = (int)Math.Round((double)current_index / (double)options.MaxBatchSize, MidpointRounding.ToNegativeInfinity);
+ int w = CRw;
+ if ((x + CRw) > source.Width)
+ {
+ w = source.Width - x;
+ }
+ int h = CRh;
+ if ((y + CRh) > source.Height)
+ {
+ h = source.Height - y;
+ }
+ // Заполнение b_index тензора в p_index батче
+ FillTensor(tensors[p_index].Tensor, source, x, y, w, h, b_index, options, pixToTensor);
+ // Указание смещений для данного тензора
+ tensors[p_index].Offsets[b_index] = new OffsetBox(x, y, options.Crop.Width, options.Crop.Height);
+ }
+ });
+ return tensors;
}
- return tensors;
}
-
// if resize only
- result = new PredictionInput[1];
+ result = new ImagePredictionInput[1];
using (var copy = image.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
{
Tensor tensor = InitInputTensor(options);
FillTensor(tensor, copy, 0, options, pixToTensor);
- result[0] = new PredictionInput { Count = 1, Offsets = null, Tensor = tensor };
+ result[0] = new ImagePredictionInput
+ {
+ Count = 1,
+ Offsets = null,
+ Tensor = tensor
+ };
}
return result;
}
+
+ private static IEnumerable SteppedIterator(int startIndex, int endIndex, int stepSize)
+ {
+ for (int i = startIndex; i < endIndex; i += stepSize)
+ {
+ yield return i;
+ }
+ }
+
public static Image Crop(Image source, float x1, float y1, float x2, float y2)
{
int left = 0;
diff --git a/ZeroLevel.NN/Services/NMS.cs b/ZeroLevel.NN/Services/NMS.cs
new file mode 100644
index 0000000..c9ccee5
--- /dev/null
+++ b/ZeroLevel.NN/Services/NMS.cs
@@ -0,0 +1,118 @@
+using ZeroLevel.NN.Models;
+
+namespace ZeroLevel.NN.Services
+{
+ public static class NMS
+ {
+ private const float IOU_THRESHOLD = .2f;
+ private const float IOU_MERGE_THRESHOLD = .5f;
+
+ public static void Apply(List boxes)
+ {
+ for (int i = 0; i < boxes.Count - 1; i++)
+ {
+ var left = boxes[i];
+ for (int j = i + 1; j < boxes.Count; j++)
+ {
+ var right = boxes[j];
+ if (left.Class == right.Class)
+ {
+ // удаление вложенных боксов
+ var ni = NestingIndex(left, right);
+ if (ni == 1)
+ {
+ boxes.RemoveAt(i);
+ i--;
+ break;
+ }
+ else if (ni == 2)
+ {
+ boxes.RemoveAt(j);
+ j--;
+ }
+ // -----------------------------
+ else
+ {
+ var iou = IOU(left, right);
+ if (iou > IOU_THRESHOLD)
+ {
+ if (left.Score > right.Score)
+ {
+ boxes.RemoveAt(j);
+ j--;
+ }
+ else
+ {
+ boxes.RemoveAt(i);
+ i--;
+ break;
+ }
+ }
+ /*else if (threshold > 0.01f && iou > 0.2f)
+ {
+ // UNION
+ var x1 = Math.Min(left.X, right.X);
+ var y1 = Math.Min(left.Y, right.Y);
+ var x2 = Math.Max(left.X + left.W, right.X + right.W);
+ var y2 = Math.Max(left.Y + left.H, right.Y + right.H);
+ var w = x2 - x1;
+ var h = y2 - y1;
+ boxes.RemoveAt(j);
+ boxes.RemoveAt(i);
+ boxes.Add(new Prediction
+ {
+ Class = left.Class,
+ Label = left.Label,
+ Score = Math.Max(left.Score, right.Score),
+ Cx = x1 + w / 2.0f,
+ Cy = y1 + h / 2.0f,
+ W = w,
+ H = h
+ });
+ i--;
+ break;
+ }*/
+ }
+ }
+ }
+ }
+ }
+
+ ///
+ /// проверка на вложенность боксов
+ ///
+ static int NestingIndex(YoloPrediction box1, YoloPrediction box2)
+ {
+ if (box1.X > box2.X &&
+ box1.Y > box2.Y &&
+ (box1.X + box1.W) < (box2.X + box2.W) &&
+ (box1.Y + box1.H) < (box2.Y + box2.H))
+ {
+ return 1;
+ }
+ if (box2.X > box1.X &&
+ box2.Y > box1.Y &&
+ (box2.X + box2.W) < (box1.X + box1.W) &&
+ (box2.Y + box2.H) < (box1.Y + box1.H))
+ {
+ return 2;
+ }
+ return 0;
+ }
+
+ static float IOU(YoloPrediction box1, YoloPrediction box2)
+ {
+ var left = (float)Math.Max(box1[0], box2[0]);
+ var right = (float)Math.Min(box1[2], box2[2]);
+
+ var top = (float)Math.Max(box1[1], box2[1]);
+ var bottom = (float)Math.Min(box1[3], box2[3]);
+
+ var width = right - left;
+ var height = bottom - top;
+ var intersectionArea = width * height;
+
+ return intersectionArea / (float)(box1.Area + box2.Area - intersectionArea);
+ }
+ }
+}
diff --git a/ZeroLevel.NN/Services/SSDNN.cs b/ZeroLevel.NN/Services/SSDNN.cs
index 87b65b1..484ae98 100644
--- a/ZeroLevel.NN/Services/SSDNN.cs
+++ b/ZeroLevel.NN/Services/SSDNN.cs
@@ -1,7 +1,6 @@
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using SixLabors.ImageSharp;
-using SixLabors.ImageSharp.PixelFormats;
using ZeroLevel.NN.Models;
namespace ZeroLevel.NN
@@ -18,6 +17,7 @@ namespace ZeroLevel.NN
try
{
var so = SessionOptions.MakeSessionOptionWithCudaProvider(0);
+ so.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_VERBOSE;
so.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL;
_session = new InferenceSession(modelPath, so);
}
@@ -64,7 +64,7 @@ namespace ZeroLevel.NN
vector[i] *= inverseLength;
}
}
- protected PredictionInput[] MakeInputBatch(Image image, ImagePreprocessorOptions options)
+ protected ImagePredictionInput[] MakeInputBatch(Image image, ImagePreprocessorOptions options)
{
return ImagePreprocessor.ToTensors(image, options);
}
@@ -75,6 +75,22 @@ namespace ZeroLevel.NN
return input[0].Tensor;
}
+ protected int Argmax(float[] embedding)
+ {
+ if (embedding.Length == 0) return -1;
+ var im = 0;
+ var max = embedding[0];
+ for (var i = 1; i < embedding.Length; i++)
+ {
+ if (embedding[i] > max)
+ {
+ im = i;
+ max = embedding[i];
+ }
+ }
+ return im;
+ }
+
public void Dispose()
{
_session?.Dispose();
diff --git a/ZeroLevel.NN/ZeroLevel.NN.csproj b/ZeroLevel.NN/ZeroLevel.NN.csproj
index 055fdc2..f6bbabf 100644
--- a/ZeroLevel.NN/ZeroLevel.NN.csproj
+++ b/ZeroLevel.NN/ZeroLevel.NN.csproj
@@ -7,7 +7,7 @@
True
embedded
none
- 1.0.0.1
+ 1.0.0.2
Ogoun
Ogoun
Copyright Ogoun 2022
@@ -15,13 +15,18 @@
https://github.com/ogoun/Zero/wiki
https://github.com/ogoun/Zero
git
- New architectures
+
+ AnyCPU;x64
False
+
+ False
+
+
True
@@ -30,10 +35,10 @@
-
-
-
-
+
+
+
+
diff --git a/ZeroLevel.SqLite/BaseSqLiteDB.cs b/ZeroLevel.SqLite/BaseSqLiteDB.cs
index ebb8b2a..a59174e 100644
--- a/ZeroLevel.SqLite/BaseSqLiteDB.cs
+++ b/ZeroLevel.SqLite/BaseSqLiteDB.cs
@@ -1,77 +1,94 @@
-using System;
-using System.Data.SQLite;
+using SQLite;
+using System;
+using System.Collections.Generic;
using System.IO;
+using System.Linq.Expressions;
using ZeroLevel.Services.FileSystem;
namespace ZeroLevel.SqLite
{
- public abstract class BaseSqLiteDB
+ public abstract class BaseSqLiteDB
+ : IDisposable
+ where T : class, new()
{
- #region Helpers
- protected static bool HasColumn(SQLiteDataReader dr, string columnName)
+ protected SQLiteConnection _db;
+ public BaseSqLiteDB(string name)
{
- for (int i = 0; i < dr.FieldCount; i++)
- {
- if (dr.GetName(i).Equals(columnName, StringComparison.InvariantCultureIgnoreCase))
- return true;
- }
- return false;
+ _db = new SQLiteConnection(PrepareDb(name));
}
- protected static Tr Read(SQLiteDataReader reader, int index)
+
+ public T Append(T record)
{
- if (reader == null || reader.FieldCount <= index || reader[index] == DBNull.Value) return default;
- Type t;
- if ((t = Nullable.GetUnderlyingType(typeof(Tr))) != null)
- {
- return (Tr)Convert.ChangeType(reader[index], t);
- }
- return (Tr)Convert.ChangeType(reader[index], typeof(Tr));
+ _db.Insert(record);
+ return record;
}
- protected static Tr Read
(SQLiteDataReader reader, string name)
+
+ public void CreateTable()
{
- if (reader == null || !HasColumn(reader, name) || reader[name] == DBNull.Value) return default;
- Type t;
- if ((t = Nullable.GetUnderlyingType(typeof(Tr))) != null)
- {
- return (Tr)Convert.ChangeType(reader[name], t);
- }
- return (Tr)Convert.ChangeType(reader[name], typeof(Tr));
+ _db.CreateTable();
}
- protected static void Execute(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null)
+ public void DropTable()
{
- using (var cmd = new SQLiteCommand(query, connection))
- {
- if (parameters != null && parameters.Length > 0)
- {
- cmd.Parameters.AddRange(parameters);
- }
- cmd.ExecuteNonQuery();
- }
+ _db.DropTable();
}
- protected static object ExecuteScalar(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null)
+ public IEnumerable SelectAll()
{
- using (var cmd = new SQLiteCommand(query, connection))
+ return _db.Table();
+ }
+
+ public IEnumerable SelectBy(Expression> predicate)
+ {
+ return _db.Table().Where(predicate);
+ }
+
+ public T Single(Expression> predicate)
+ {
+ return _db.Table().FirstOrDefault(predicate);
+ }
+
+ public T Single(Expression> predicate, Expression> orderBy, bool desc = false)
+ {
+ if (desc)
{
- if (parameters != null && parameters.Length > 0)
- {
- cmd.Parameters.AddRange(parameters);
- }
- return cmd.ExecuteScalar();
+ return _db.Table().Where(predicate).OrderByDescending(orderBy).FirstOrDefault();
}
+ return _db.Table().Where(predicate).OrderBy(orderBy).FirstOrDefault();
}
- protected static SQLiteDataReader Read(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null)
+ public T Single(Expression> orderBy, bool desc = false)
{
- using (var cmd = new SQLiteCommand(query, connection))
+ if (desc)
{
- if (parameters != null && parameters.Length > 0)
- {
- cmd.Parameters.AddRange(parameters);
- }
- return cmd.ExecuteReader();
+ return _db.Table().OrderByDescending(orderBy).FirstOrDefault();
}
+ return _db.Table().OrderBy(orderBy).FirstOrDefault();
+ }
+
+ public IEnumerable SelectBy(int N, Expression> predicate)
+ {
+ return _db.Table().Where(predicate).Take(N);
+ }
+
+ public long Count()
+ {
+ return _db.Table().Count();
+ }
+
+ public long Count(Expression> predicate)
+ {
+ return _db.Table().Count(predicate);
+ }
+
+ public int Delete(Expression> predicate)
+ {
+ return _db.Table().Delete(predicate);
+ }
+
+ public void Update(T record)
+ {
+ _db.Update(record);
}
protected static string PrepareDb(string path)
@@ -80,13 +97,23 @@ namespace ZeroLevel.SqLite
{
path = Path.Combine(FSUtils.GetAppLocalDbDirectory(), path);
}
- if (!File.Exists(path))
- {
- SQLiteConnection.CreateFile(path);
- }
return Path.GetFullPath(path);
}
- #endregion Helpers
+ protected abstract void DisposeStorageData();
+
+ public void Dispose()
+ {
+ DisposeStorageData();
+ try
+ {
+ _db?.Close();
+ _db?.Dispose();
+ }
+ catch (Exception ex)
+ {
+ Log.Error(ex, "[BaseSqLiteDB] Fault close db connection");
+ }
+ }
}
}
diff --git a/ZeroLevel.SqLite/Model/User.cs b/ZeroLevel.SqLite/Model/User.cs
index f944657..fd75e96 100644
--- a/ZeroLevel.SqLite/Model/User.cs
+++ b/ZeroLevel.SqLite/Model/User.cs
@@ -1,10 +1,15 @@
-namespace ZeroLevel.SqLite
+using SQLite;
+
+namespace ZeroLevel.SqLite
{
public class User
{
+ [PrimaryKey, AutoIncrement]
public long Id { get; set; }
+ [Indexed]
public string UserName { get; set; }
public string DisplayName { get; set; }
+ [Indexed]
public byte[] PasswordHash { get; set; }
public long Timestamp { get; set; }
public long Creator { get; set; }
diff --git a/ZeroLevel.SqLite/SqLiteDelayDataStorage.cs b/ZeroLevel.SqLite/SqLiteDelayDataStorage.cs
index 943dc6c..301b7f6 100644
--- a/ZeroLevel.SqLite/SqLiteDelayDataStorage.cs
+++ b/ZeroLevel.SqLite/SqLiteDelayDataStorage.cs
@@ -1,24 +1,28 @@
-using System;
-using System.Collections.Generic;
-using System.Data.SQLite;
-using System.IO;
-using System.Text;
+using SQLite;
+using System;
using System.Threading;
using ZeroLevel.Services.Serialization;
using ZeroLevel.Services.Shedulling;
namespace ZeroLevel.SqLite
{
+ public sealed class ExpirationRecord
+ {
+ [PrimaryKey, AutoIncrement]
+ public long Id { get; set; }
+ [Indexed]
+ public long Expiration { get; set; }
+ public byte[] Data { get; set; }
+ }
+
public sealed class SqLiteDelayDataStorage
- : BaseSqLiteDB, IDisposable
- where T : IBinarySerializable
+ : BaseSqLiteDB
+ where T : class, IBinarySerializable, new()
{
#region Fields
private readonly IExpirationSheduller _sheduller;
private readonly Func _expire_date_calc_func;
- private readonly SQLiteConnection _db;
- private readonly string _table_name;
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
#endregion Fields
@@ -28,14 +32,10 @@ namespace ZeroLevel.SqLite
public SqLiteDelayDataStorage(string database_file_path,
Func expire_callback,
Func expire_date_calc_func)
+ : base(database_file_path)
{
this._expire_date_calc_func = expire_date_calc_func;
- var path = PrepareDb(database_file_path);
- _table_name = "expiration";
- _db = new SQLiteConnection($"Data Source={path};Version=3;");
- _db.Open();
- Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB, expirationtime INTEGER)", _db);
- Execute($"CREATE INDEX IF NOT EXISTS expirationtime_index ON {_table_name} (expirationtime)", _db);
+ CreateTable();
_sheduller = Sheduller.CreateExpirationSheduller();
OnExpire += expire_callback;
Preload();
@@ -64,13 +64,8 @@ namespace ZeroLevel.SqLite
long id = -1;
try
{
- Execute($"INSERT INTO {_table_name} ('body', 'expirationtime') values (@body, @expirationtime)", _db,
- new SQLiteParameter[]
- {
- new SQLiteParameter("body", MessageSerializer.Serialize(packet)),
- new SQLiteParameter("expirationtime", expirationTime)
- });
- id = (long)ExecuteScalar("select last_insert_rowid();", _db);
+ var r = Append(new ExpirationRecord { Expiration = expirationTime, Data = MessageSerializer.Serialize(packet) });
+ id = r.Id;
}
catch (Exception ex)
{
@@ -91,17 +86,13 @@ namespace ZeroLevel.SqLite
private void Preload()
{
- SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
- reader = Read($"SELECT id, expirationtime FROM {_table_name}", _db);
- while (reader.Read())
+ foreach (var record in SelectAll())
{
- var id = reader.GetInt64(0);
- _sheduller.Push(new DateTime(reader.GetInt64(1), DateTimeKind.Local), (k) => Pop(id));
+ _sheduller.Push(new DateTime(record.Expiration, DateTimeKind.Local), (k) => Pop(record.Id));
}
- reader.Close();
}
catch (Exception ex)
{
@@ -111,7 +102,6 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
- reader = null;
}
private void Pop(long id)
@@ -122,7 +112,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterReadLock();
try
{
- body = (byte[])ExecuteScalar($"SELECT body FROM {_table_name} WHERE id=@id", _db, new SQLiteParameter[] { new SQLiteParameter("id", id) });
+ body = Single(r=>r.Id == id)?.Data;
}
catch (Exception ex)
{
@@ -161,8 +151,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
- Execute($"DELETE FROM {_table_name} WHERE id = @id", _db,
- new SQLiteParameter[] { new SQLiteParameter("id", id) });
+ Delete(r => r.Id == id);
}
catch (Exception ex)
{
@@ -178,17 +167,8 @@ namespace ZeroLevel.SqLite
#region IDisposable
- public void Dispose()
+ protected override void DisposeStorageData()
{
- try
- {
- _db?.Close();
- _db?.Dispose();
- }
- catch (Exception ex)
- {
- Log.Error(ex, "[SqLiteDelayDataStorage] Fault close db connection");
- }
_sheduller.Dispose();
}
diff --git a/ZeroLevel.SqLite/SqLiteDupStorage.cs b/ZeroLevel.SqLite/SqLiteDupStorage.cs
index 84d83ea..117754b 100644
--- a/ZeroLevel.SqLite/SqLiteDupStorage.cs
+++ b/ZeroLevel.SqLite/SqLiteDupStorage.cs
@@ -1,6 +1,6 @@
-using System;
+using SQLite;
+using System;
using System.Collections.Generic;
-using System.Data.SQLite;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
@@ -8,40 +8,37 @@ using System.Threading;
namespace ZeroLevel.SqLite
{
+ public sealed class DuplicateRecord
+ {
+ [PrimaryKey, AutoIncrement]
+ public long Id { get; set; }
+ [Indexed]
+ public string Hash { get; set; }
+ public long Timestamp { get; set; }
+ public byte[] Data { get; set; }
+ }
+
///
/// Хранит данные указанное число дней, и позволяет выполнить проверку наличия данных, для отбрасывания дубликатов
///
public sealed class SqLiteDupStorage
- : BaseSqLiteDB, IDisposable
+ : BaseSqLiteDB
{
#region Fields
- private const string DEFAUL_TABLE_NAME = "History";
-
- private readonly SQLiteConnection _db;
private readonly long _removeOldRecordsTaskKey;
private readonly int _countDays;
- private readonly string _table_name;
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
#endregion Fields
#region Private members
-
- private sealed class DuplicationStorageRecord
- {
- public string Hash { get; set; }
- public byte[] Body { get; set; }
- public long Timestamp { get; set; }
- }
-
private void RemoveOldRecordsTask(long key)
{
_rwLock.EnterWriteLock();
try
{
- Execute($"DELETE FROM {_table_name} WHERE timestamp < @limit", _db,
- new SQLiteParameter[] { new SQLiteParameter("limit", DateTime.Now.AddDays(-_countDays).Ticks) });
+ Delete(r => r.Timestamp < DateTime.Now.AddDays(-_countDays).Ticks);
}
catch (Exception ex)
{
@@ -57,21 +54,10 @@ namespace ZeroLevel.SqLite
#region Ctor
- public SqLiteDupStorage(string database_file_path, string tableName, int period)
+ public SqLiteDupStorage(string database_file_path, int period)
+ : base(database_file_path)
{
- var path = PrepareDb(database_file_path);
- if (string.IsNullOrWhiteSpace(tableName))
- {
- _table_name = DEFAUL_TABLE_NAME;
- }
- else
- {
- _table_name = tableName;
- }
- _db = new SQLiteConnection($"Data Source={path};Version=3;");
- _db.Open();
- Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, body BLOB, timestamp INTEGER)", _db);
- Execute($"CREATE INDEX IF NOT EXISTS hash_index ON {_table_name} (hash)", _db);
+ CreateTable();
_countDays = period > 0 ? period : 1;
_removeOldRecordsTaskKey = Sheduller.RemindEvery(TimeSpan.FromMinutes(1), RemoveOldRecordsTask);
}
@@ -87,17 +73,14 @@ namespace ZeroLevel.SqLite
{
var hash = GenerateSHA256String(body);
var timestamp = DateTime.Now.Ticks;
- SQLiteDataReader reader;
_rwLock.EnterReadLock();
var exists = new List();
try
{
- reader = Read($"SELECT body FROM {_table_name} WHERE hash=@hash", _db, new SQLiteParameter[] { new SQLiteParameter("hash", hash) });
- while (reader.Read())
+ foreach (var record in SelectBy(r => r.Hash == hash))
{
- exists.Add((byte[])reader.GetValue(0));
+ exists.Add(record.Data);
}
- reader.Close();
}
catch (Exception ex)
{
@@ -108,7 +91,6 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
- reader = null;
if (exists.Any())
{
foreach (var candidate in exists)
@@ -120,13 +102,12 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
- Execute($"INSERT INTO {_table_name} ('hash', 'body', 'timestamp') values (@hash, @body, @timestamp)", _db,
- new SQLiteParameter[]
- {
- new SQLiteParameter("hash", hash),
- new SQLiteParameter("body", body),
- new SQLiteParameter("timestamp", timestamp)
- });
+ Append(new DuplicateRecord
+ {
+ Data = body,
+ Hash = hash,
+ Timestamp = timestamp
+ });
}
catch (Exception ex)
{
@@ -143,17 +124,9 @@ namespace ZeroLevel.SqLite
#region IDisposable
- public void Dispose()
+ protected override void DisposeStorageData()
{
Sheduller.Remove(_removeOldRecordsTaskKey);
- try
- {
- _db?.Dispose();
- }
- catch (Exception ex)
- {
- Log.Error(ex, "[SQLiteDupStorage] Fault close db connection");
- }
}
#endregion IDisposable
diff --git a/ZeroLevel.SqLite/SqLitePacketBuffer.cs b/ZeroLevel.SqLite/SqLitePacketBuffer.cs
index e20da5d..d17de22 100644
--- a/ZeroLevel.SqLite/SqLitePacketBuffer.cs
+++ b/ZeroLevel.SqLite/SqLitePacketBuffer.cs
@@ -1,15 +1,25 @@
-using System;
-using System.Data.SQLite;
+using SQLite;
+using System;
using System.Threading;
using ZeroLevel.Services.Serialization;
namespace ZeroLevel.SqLite
{
+
+ public sealed class PacketRecord
+ {
+ [PrimaryKey, AutoIncrement]
+ public long Id { get; set; }
+ [Indexed]
+ public long Timestamp { get; set; }
+ public byte[] Data { get; set; }
+ }
+
///
/// Промежуточное/временное хранилище пакетов данных, для случаев сбоя доставок через шину данных
///
public sealed class SqLitePacketBuffer
- : BaseSqLiteDB, IDisposable
+ : BaseSqLiteDB
where T : IBinarySerializable
{
private sealed class PacketBufferRecord
@@ -20,20 +30,14 @@ namespace ZeroLevel.SqLite
#region Fields
- private readonly SQLiteConnection _db;
- private readonly string _table_name;
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
#endregion Fields
public SqLitePacketBuffer(string database_file_path)
+ : base(database_file_path)
{
- var path = PrepareDb(database_file_path);
- _table_name = "packets";
- _db = new SQLiteConnection($"Data Source={path};Version=3;");
- _db.Open();
- Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB, created INTEGER)", _db);
- Execute($"CREATE INDEX IF NOT EXISTS created_index ON {_table_name} (created)", _db);
+ CreateTable();
}
public void Push(T frame)
@@ -43,13 +47,11 @@ namespace ZeroLevel.SqLite
var creationTime = DateTime.Now.Ticks;
try
{
- Execute($"INSERT INTO {_table_name} ('body', 'created') values (@body, @created)", _db,
- new SQLiteParameter[]
- {
- new SQLiteParameter("body", MessageSerializer.Serialize(frame)),
- new SQLiteParameter("created", creationTime)
- });
- id = (long)ExecuteScalar("select last_insert_rowid();", _db);
+ id = Append(new PacketRecord
+ {
+ Data = MessageSerializer.Serialize(frame),
+ Timestamp = creationTime
+ }).Id;
}
catch (Exception ex)
{
@@ -65,25 +67,20 @@ namespace ZeroLevel.SqLite
{
bool success = false;
long id = -1;
- SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
- reader = Read($"SELECT id, body FROM {_table_name} ORDER BY created ASC LIMIT 1", _db);
- if (reader.Read())
+ var record = Single(r => r.Timestamp);
+ id = record.Id;
+ var body = record.Data;
+ try
+ {
+ success = pop_callback(MessageSerializer.Deserialize(body));
+ }
+ catch (Exception ex)
{
- id = reader.GetInt64(0);
- var body = (byte[])reader.GetValue(1);
- try
- {
- success = pop_callback(MessageSerializer.Deserialize(body));
- }
- catch (Exception ex)
- {
- Log.Error(ex, "Fault handle buffered data");
- }
+ Log.Error(ex, "Fault handle buffered data");
}
- reader.Close();
}
catch (Exception ex)
{
@@ -97,7 +94,6 @@ namespace ZeroLevel.SqLite
{
RemoveRecordById(id);
}
- reader = null;
return success;
}
@@ -106,8 +102,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
- Execute($"DELETE FROM {_table_name} WHERE id = @id", _db,
- new SQLiteParameter[] { new SQLiteParameter("id", id) });
+ Delete(r => r.Id == id);
}
catch (Exception ex)
{
@@ -119,17 +114,8 @@ namespace ZeroLevel.SqLite
}
}
- public void Dispose()
+ protected override void DisposeStorageData()
{
- try
- {
- _db?.Close();
- _db?.Dispose();
- }
- catch (Exception ex)
- {
- Log.Error(ex, "[SqLitePacketBuffer] Fault close db connection");
- }
}
}
}
diff --git a/ZeroLevel.SqLite/SqLiteUserRepository.cs b/ZeroLevel.SqLite/SqLiteUserRepository.cs
index 168a3a4..d39a706 100644
--- a/ZeroLevel.SqLite/SqLiteUserRepository.cs
+++ b/ZeroLevel.SqLite/SqLiteUserRepository.cs
@@ -1,18 +1,15 @@
using System;
using System.Collections.Generic;
-using System.Data.SQLite;
using System.Threading;
using ZeroLevel.Models;
namespace ZeroLevel.SqLite
{
public class SqLiteUserRepository
- : BaseSqLiteDB
+ : BaseSqLiteDB
{
#region Fields
- private readonly SQLiteConnection _db;
- private readonly string _table_name = "users";
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
#endregion Fields
@@ -20,13 +17,8 @@ namespace ZeroLevel.SqLite
#region Ctor
public SqLiteUserRepository()
+ : base("users.db")
{
- var path =PrepareDb("users.db");
- _db = new SQLiteConnection($"Data Source={path};Version=3;");
- _db.Open();
- Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT, displayname TEXT, hash BLOB, timestamp INTEGER, creator INTEGER, role INTEGER)", _db);
- Execute($"CREATE INDEX IF NOT EXISTS username_index ON {_table_name} (username)", _db);
- Execute($"CREATE INDEX IF NOT EXISTS hash_index ON {_table_name} (hash)", _db);
}
#endregion Ctor
@@ -34,25 +26,13 @@ namespace ZeroLevel.SqLite
public IEnumerable GetAll()
{
var list = new List();
- SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
- reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name}", _db);
- while (reader.Read())
+ foreach (var r in SelectAll())
{
- list.Add(new User
- {
- Id = reader.GetInt64(0),
- UserName = reader.GetString(1),
- DisplayName = Read(reader, 2),
- PasswordHash = (byte[])reader.GetValue(3),
- Timestamp = reader.GetInt64(4),
- Creator = reader.GetInt64(5),
- Role = (UserRole)reader.GetInt32(6)
- });
+ list.Add(r);
}
- reader.Close();
}
catch (Exception ex)
{
@@ -62,34 +42,16 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
- reader = null;
return list;
}
public User Get(long id)
{
User user = null;
- SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
- reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name} WHERE id = @id", _db,
- new SQLiteParameter[] { new SQLiteParameter("id", id) });
- if (reader.Read())
- {
- var body = (byte[])reader.GetValue(1);
- user = new User
- {
- Id = reader.GetInt64(0),
- UserName = reader.GetString(1),
- DisplayName = reader.GetString(2),
- PasswordHash = (byte[])reader.GetValue(3),
- Timestamp = reader.GetInt64(4),
- Creator = reader.GetInt64(5),
- Role = (UserRole)reader.GetInt32(6)
- };
- }
- reader.Close();
+ user = Single(r => r.Id == id);
}
catch (Exception ex)
{
@@ -99,37 +61,16 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
- reader = null;
return user;
}
public User Get(string username, byte[] hash)
{
User user = null;
- SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
- reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name} WHERE username = @username AND hash = @hash", _db,
- new SQLiteParameter[]
- {
- new SQLiteParameter("username", username),
- new SQLiteParameter("hash", hash)
- });
- if (reader.Read())
- {
- user = new User
- {
- Id = reader.GetInt64(0),
- UserName = reader.GetString(1),
- DisplayName = reader.GetString(2),
- PasswordHash = (byte[])reader.GetValue(3),
- Timestamp = reader.GetInt64(4),
- Creator = reader.GetInt64(5),
- Role = (UserRole)reader.GetInt32(6)
- };
- }
- reader.Close();
+ user = Single(r => r.UserName == username && r.PasswordHash == hash);
}
catch (Exception ex)
{
@@ -139,7 +80,6 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
- reader = null;
return user;
}
@@ -150,22 +90,12 @@ namespace ZeroLevel.SqLite
var creationTime = DateTime.UtcNow.Ticks;
try
{
- var count_obj = ExecuteScalar($"SELECT COUNT(*) FROM {_table_name} WHERE username=@username", _db, new SQLiteParameter[] { new SQLiteParameter("username", user.UserName) });
- if (count_obj != null && (long)count_obj > 0)
+ var count_obj = Count(r => r.UserName == user.UserName);
+ if (count_obj > 0)
{
- return InvokeResult.Fault("Пользователь уже существует");
+ return InvokeResult.Fault("Пользователь с таким именем уже существует");
}
- Execute($"INSERT INTO {_table_name} ('username', 'displayname', 'hash', 'timestamp', 'creator', 'role') values (@username, @displayname, @hash, @timestamp, @creator, @role)", _db,
- new SQLiteParameter[]
- {
- new SQLiteParameter("username", user.UserName),
- new SQLiteParameter("displayname", user.DisplayName),
- new SQLiteParameter("hash", user.PasswordHash),
- new SQLiteParameter("timestamp", creationTime),
- new SQLiteParameter("creator", user.Creator),
- new SQLiteParameter("role", user.Role)
- });
- id = (long)ExecuteScalar("select last_insert_rowid();", _db);
+ id = Append(user).Id;
}
catch (Exception ex)
{
@@ -184,8 +114,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
- Execute($"DELETE FROM {_table_name} WHERE username = @username", _db,
- new SQLiteParameter[] { new SQLiteParameter("username", login) });
+ Delete(r => r.UserName == login);
return InvokeResult.Succeeding();
}
catch (Exception ex)
@@ -198,5 +127,9 @@ namespace ZeroLevel.SqLite
_rwLock.ExitWriteLock();
}
}
+
+ protected override void DisposeStorageData()
+ {
+ }
}
}
diff --git a/ZeroLevel.SqLite/UserCacheRepository.cs b/ZeroLevel.SqLite/UserCacheRepository.cs
index f48c2b4..569bc90 100644
--- a/ZeroLevel.SqLite/UserCacheRepository.cs
+++ b/ZeroLevel.SqLite/UserCacheRepository.cs
@@ -1,36 +1,36 @@
-using System;
+using SQLite;
+using System;
using System.Collections.Generic;
-using System.Data.SQLite;
using System.Runtime.CompilerServices;
using System.Threading;
using ZeroLevel.Services.Serialization;
namespace ZeroLevel.SqLite
{
+ public sealed class DataRecord
+ {
+ [PrimaryKey, AutoIncrement]
+ public long Id { get; set; }
+ [Indexed]
+ public string Key { get; set; }
+ public byte[] Data { get; set; }
+ }
public sealed class UserCacheRepository
- : BaseSqLiteDB, IDisposable
+ : BaseSqLiteDB
where T : IBinarySerializable
{
#region Fields
- private readonly SQLiteConnection _db;
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
- private readonly string _tableName;
#endregion Fields
#region Ctor
public UserCacheRepository()
+ : base(typeof(T).Name)
{
- _tableName = typeof(T).Name;
-
- var path = PrepareDb($"{_tableName}_user_cachee.db");
- _db = new SQLiteConnection($"Data Source={path};Version=3;");
- _db.Open();
-
- Execute($"CREATE TABLE IF NOT EXISTS {_tableName} (id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT, body BLOB)", _db);
- Execute($"CREATE INDEX IF NOT EXISTS key_index ON {_tableName} (key)", _db);
+ CreateTable();
}
#endregion Ctor
@@ -47,8 +47,8 @@ namespace ZeroLevel.SqLite
_rwLock.EnterReadLock();
try
{
- var count_obj = ExecuteScalar($"SELECT COUNT(*) FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[] { new SQLiteParameter("key", key) });
- if (count_obj != null && (long)count_obj > 0)
+ var count_obj = Count(r => r.Key == key);
+ if (count_obj > 0)
{
update = true;
}
@@ -68,20 +68,16 @@ namespace ZeroLevel.SqLite
var body = MessageSerializer.Serialize(data);
if (update)
{
- Execute($"UPDATE {_tableName} SET body=@body WHERE key=@key", _db,
- new SQLiteParameter[]
- {
- new SQLiteParameter("key", key),
- new SQLiteParameter("body", body)
- });
+ var r = Single(r => r.Key == key);
+ r.Data = body;
+ Update(r);
}
else
{
- Execute($"INSERT INTO {_tableName} ('key', 'body') values (@key, @body)", _db,
- new SQLiteParameter[]
+ Append(new DataRecord
{
- new SQLiteParameter("key", key),
- new SQLiteParameter("body", body)
+ Data = body,
+ Key = key
});
}
return true;
@@ -103,10 +99,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
- Execute($"DELETE FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[]
- {
- new SQLiteParameter("key", key)
- });
+ Delete(r => r.Key == key);
}
catch (Exception ex)
{
@@ -124,10 +117,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
- return Convert.ToInt64(ExecuteScalar($"SELECT COUNT(*) FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[]
- {
- new SQLiteParameter("key", key)
- }));
+ return Count(r => r.Key == key);
}
catch (Exception ex)
{
@@ -144,23 +134,17 @@ namespace ZeroLevel.SqLite
{
var key = KEY(userid, name);
var result = new List();
- SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
- reader = Read($"SELECT [body] FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[]
+ foreach (var r in SelectBy(r=>r.Key == key))
{
- new SQLiteParameter("key", key)
- });
- while (reader.Read())
- {
- var data = Read(reader, 0);
+ var data = r.Data;
if (data != null)
{
result.Add(MessageSerializer.Deserialize(data));
}
}
- reader.Close();
}
catch (Exception ex)
{
@@ -169,7 +153,6 @@ namespace ZeroLevel.SqLite
finally
{
_rwLock.ExitReadLock();
- reader = null;
}
return result;
}
@@ -177,17 +160,9 @@ namespace ZeroLevel.SqLite
#endregion API
#region IDisposable
-
- public void Dispose()
+
+ protected override void DisposeStorageData()
{
- try
- {
- _db?.Dispose();
- }
- catch (Exception ex)
- {
- Log.Error(ex, "[UserCacheRepository] Fault close db connection");
- }
}
#endregion IDisposable
diff --git a/ZeroLevel.SqLite/ZeroLevel.SqLite.csproj b/ZeroLevel.SqLite/ZeroLevel.SqLite.csproj
index c816c7c..3f601f3 100644
--- a/ZeroLevel.SqLite/ZeroLevel.SqLite.csproj
+++ b/ZeroLevel.SqLite/ZeroLevel.SqLite.csproj
@@ -13,9 +13,9 @@ Based on System.Data.SQLite.Core
AnyCPU;x64;x86
Move to new ZeroLevel version
- 1.0.3.0
- 1.0.3.0
- 1.0.3.0
+ 1.0.4.0
+ 1.0.4.0
+ 1.0.4.0
Library
@@ -27,11 +27,17 @@ Based on System.Data.SQLite.Core
-
+
+
+
+ PreserveNewest
+
+
+
diff --git a/ZeroLevel.SqLite/sqlite3.dll b/ZeroLevel.SqLite/sqlite3.dll
new file mode 100644
index 0000000..70a8f7f
Binary files /dev/null and b/ZeroLevel.SqLite/sqlite3.dll differ
diff --git a/ZeroLevel.UnitTests/SuffixAutomataTests.cs b/ZeroLevel.UnitTests/SuffixAutomataTests.cs
new file mode 100644
index 0000000..8d20a38
--- /dev/null
+++ b/ZeroLevel.UnitTests/SuffixAutomataTests.cs
@@ -0,0 +1,42 @@
+using Xunit;
+using ZeroLevel.Semantic;
+using ZeroLevel.Services.Text;
+
+namespace ZeroLevel.UnitTests
+{
+ public class SuffixAutomataTests
+ {
+ [Fact]
+ public void IsSubstringTest()
+ {
+ var text = @"=Однако эту оценку легко показать и без знания алгоритма. Вспомним о том, что число состояний равно количеству различных значений множеств endpos.#";
+ var automata = new SuffixAutomata();
+ automata.Init();
+ foreach (var ch in text)
+ {
+ automata.Extend(ch);
+ }
+ Assert.True(automata.IsSubstring("Вспомним"));
+ Assert.True(automata.IsSubstring("")); // empty line
+ Assert.True(automata.IsSubstring("#")); // end line
+ Assert.True(automata.IsSubstring("=")); // start line
+ Assert.True(automata.IsSubstring(null)); // null
+ Assert.False(automata.IsSubstring("равноценно"));
+ Assert.False(automata.IsSubstring("нетслова"));
+ }
+
+ [Fact]
+ public void IntersectionTest()
+ {
+ var text = @"=Однако эту оценку легко показать и без знания алгоритма. Вспомним о том, что число состояний равно количеству различных значений множеств endpos.#";
+ var i = LongestCommonSubstring.LCS(text, "енк");
+ Assert.Equal("енк", i);
+
+ i = LongestCommonSubstring.LCS(text, "стоя");
+ Assert.Equal("стоя", i);
+
+ i = LongestCommonSubstring.LCS(text, "горизонт");
+ Assert.Equal("гори", i);
+ }
+ }
+}
diff --git a/ZeroLevel.sln b/ZeroLevel.sln
index 8d14039..229af50 100644
--- a/ZeroLevel.sln
+++ b/ZeroLevel.sln
@@ -39,8 +39,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "WPFExamples", "WPFExamples"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ConnectionTest", "ConnectionTest", "{D5207A5A-2F27-4992-9BA5-0BDCFC59F133}"
EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Client", "ConnectionTest\Client\Client.csproj", "{08CDD42E-E324-40A4-88C3-EDD0493AAF84}"
-EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Server", "ConnectionTest\Server\Server.csproj", "{3496A688-0749-48C2-BD60-ABB42A5C17C9}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ZeroLevel.Qdrant", "ZeroLevel.Qdrant\ZeroLevel.Qdrant.csproj", "{7188B89E-96EB-4EFB-AAFB-D0A823031F99}"
@@ -63,7 +61,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Source", "TestPipeLine\Sour
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Watcher", "TestPipeLine\Watcher\Watcher.csproj", "{F70842E7-9A1D-4CC4-9F55-0953AEC9C7C8}"
EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ZeroLevel.NN", "ZeroLevel.NN\ZeroLevel.NN.csproj", "{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ZeroLevel.NN", "ZeroLevel.NN\ZeroLevel.NN.csproj", "{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Client", "ConnectionTest\Client\Client.csproj", "{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AutoLoader", "AutoLoader", "{2EF83101-63BC-4397-A005-A747189143D4}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AppContainer", "AutoLoader\AppContainer\AppContainer.csproj", "{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -171,18 +175,6 @@ Global
{04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x64.Build.0 = Release|x64
{04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x86.ActiveCfg = Release|x86
{04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x86.Build.0 = Release|x86
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|Any CPU.Build.0 = Debug|Any CPU
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x64.ActiveCfg = Debug|x64
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x64.Build.0 = Debug|x64
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x86.ActiveCfg = Debug|x86
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x86.Build.0 = Debug|x86
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|Any CPU.ActiveCfg = Release|Any CPU
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|Any CPU.Build.0 = Release|Any CPU
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x64.ActiveCfg = Release|x64
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x64.Build.0 = Release|x64
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x86.ActiveCfg = Release|x86
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x86.Build.0 = Release|x86
{3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|x64.ActiveCfg = Debug|x64
@@ -327,12 +319,35 @@ Global
{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x64.Build.0 = Release|Any CPU
{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x86.ActiveCfg = Release|Any CPU
{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x86.Build.0 = Release|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x64.Build.0 = Debug|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x86.Build.0 = Debug|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|Any CPU.Build.0 = Release|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x64.ActiveCfg = Release|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x64.Build.0 = Release|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x86.ActiveCfg = Release|Any CPU
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x86.Build.0 = Release|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x64.Build.0 = Debug|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x86.Build.0 = Debug|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|Any CPU.Build.0 = Release|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x64.ActiveCfg = Release|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x64.Build.0 = Release|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x86.ActiveCfg = Release|Any CPU
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
- {08CDD42E-E324-40A4-88C3-EDD0493AAF84} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133}
{3496A688-0749-48C2-BD60-ABB42A5C17C9} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133}
{54BB2BA9-DAC0-4162-8BC0-E4A9B898CBB0} = {FC074553-5D9F-4DF1-9130-7092E37DE768}
{3D0FE0BA-F7B1-4A63-BBA4-C96514A68426} = {FC074553-5D9F-4DF1-9130-7092E37DE768}
@@ -340,6 +355,8 @@ Global
{5CCFF557-C91F-4DD7-9530-D76FE517DA98} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15}
{82202433-6426-4737-BAB2-473AC1F74C5D} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15}
{F70842E7-9A1D-4CC4-9F55-0953AEC9C7C8} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15}
+ {2C33D5A3-6CD4-4AAA-A716-B3CD65036E25} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133}
+ {9DE345EA-955B-41A8-93AF-277C0B5A9AC5} = {2EF83101-63BC-4397-A005-A747189143D4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A65DB16F-877D-4586-A9F3-8BBBFBAF5CEB}
diff --git a/ZeroLevel/Services/Extensions/LinqExtension.cs b/ZeroLevel/Services/Extensions/LinqExtension.cs
index 5a0a5af..704e08c 100644
--- a/ZeroLevel/Services/Extensions/LinqExtension.cs
+++ b/ZeroLevel/Services/Extensions/LinqExtension.cs
@@ -4,6 +4,21 @@ namespace System.Linq
{
public static class LinqExtension
{
+ public static void ThrowIfNull(this T val, string message = null)
+ {
+ if (null == val)
+ throw new ArgumentNullException(message);
+ }
+
+ public static void Apply(this IEnumerable seq, Action action)
+ {
+ seq.ThrowIfNull();
+ action.ThrowIfNull();
+ foreach (var e in seq)
+ {
+ action.Invoke(e);
+ }
+ }
public static IEnumerable ZipLongest(this IEnumerable left, IEnumerable right)
{
IEnumerator leftEnumerator = left.GetEnumerator();
diff --git a/ZeroLevel/Services/FileSystem/FSUtils.cs b/ZeroLevel/Services/FileSystem/FSUtils.cs
index 98f6e9a..3db7ff4 100644
--- a/ZeroLevel/Services/FileSystem/FSUtils.cs
+++ b/ZeroLevel/Services/FileSystem/FSUtils.cs
@@ -341,6 +341,27 @@ namespace ZeroLevel.Services.FileSystem
var zipFile = Path.Combine(tmp.FullName, "zip.zip");
File.WriteAllBytes(zipFile, data);
ZipFile.ExtractToDirectory(zipFile, targetFolder);
+ File.Delete(zipFile);
+ }
+
+ public static void UnPackFolder(string zipFile, string targetFolder)
+ {
+ if (Directory.Exists(targetFolder))
+ {
+ try
+ {
+ FSUtils.RemoveFolder(targetFolder, 3, 3000);
+ }
+ catch (Exception ex)
+ {
+ Log.SystemError(ex, $"[FSUtils] Fault clean folder '{Path.GetDirectoryName(targetFolder)}'");
+ }
+ }
+ if (Directory.Exists(targetFolder) == false)
+ {
+ Directory.CreateDirectory(targetFolder);
+ }
+ ZipFile.ExtractToDirectory(zipFile, targetFolder);
}
public static void CopyDir(string sourceFolder, string targetFolder)
diff --git a/ZeroLevel/Services/HashFunctions/StringHash.cs b/ZeroLevel/Services/HashFunctions/StringHash.cs
index fb75a5c..9fddb0a 100644
--- a/ZeroLevel/Services/HashFunctions/StringHash.cs
+++ b/ZeroLevel/Services/HashFunctions/StringHash.cs
@@ -6,22 +6,23 @@
///
public static class StringHash
{
- public static uint DotNetFullHash(string str)
+ public static int DotNetFullHash(string str)
{
unchecked
{
int hash1 = (5381 << 16) + 5381;
int hash2 = hash1;
-
- for (int i = 0; i < str.Length; i += 2)
+ if (str != null)
{
- hash1 = ((hash1 << 5) + hash1) ^ str[i];
- if (i == str.Length - 1)
- break;
- hash2 = ((hash2 << 5) + hash2) ^ str[i + 1];
+ for (int i = 0; i < str.Length; i += 2)
+ {
+ hash1 = ((hash1 << 5) + hash1) ^ str[i];
+ if (i == str.Length - 1)
+ break;
+ hash2 = ((hash2 << 5) + hash2) ^ str[i + 1];
+ }
}
-
- return (uint)(hash1 + (hash2 * 1566083941)) & 0x7FFFFFFF;
+ return (hash1 + (hash2 * 1566083941)) & 0x7FFFFFFF;
}
}
diff --git a/ZeroLevel/Services/MemoryPools/JetPool.cs b/ZeroLevel/Services/MemoryPools/JetPool.cs
index d49e5e0..411d344 100644
--- a/ZeroLevel/Services/MemoryPools/JetPool.cs
+++ b/ZeroLevel/Services/MemoryPools/JetPool.cs
@@ -15,4 +15,15 @@ namespace MemoryPools
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Return(T instance) => _freeObjectsQueue.Push(instance);
}
+
+ public class JetValPool
+ {
+ private readonly JetStack _freeObjectsQueue = new JetStack();
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public T Get() => _freeObjectsQueue.Count > 0 ? _freeObjectsQueue.Pop() : default(T);
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public void Return(T instance) => _freeObjectsQueue.Push(instance);
+ }
}
diff --git a/ZeroLevel/Services/Network/Contracts/ISocketClient.cs b/ZeroLevel/Services/Network/Contracts/ISocketClient.cs
index 5cee702..a2c0480 100644
--- a/ZeroLevel/Services/Network/Contracts/ISocketClient.cs
+++ b/ZeroLevel/Services/Network/Contracts/ISocketClient.cs
@@ -13,8 +13,8 @@ namespace ZeroLevel.Network
IRouter Router { get; }
- bool Send(Frame data);
- bool Request(Frame data, Action callback, Action fail = null);
- bool Response(byte[] data, int identity);
+ void Send(Frame data);
+ void Request(Frame data, Action callback, Action fail = null);
+ void Response(byte[] data, int identity);
}
}
diff --git a/ZeroLevel/Services/Network/ExClient.cs b/ZeroLevel/Services/Network/ExClient.cs
index 81bc9cf..08ce2e3 100644
--- a/ZeroLevel/Services/Network/ExClient.cs
+++ b/ZeroLevel/Services/Network/ExClient.cs
@@ -22,7 +22,8 @@ namespace ZeroLevel.Network
{
try
{
- return _client.Send(FrameFactory.Create(inbox));
+ _client.Send(FrameFactory.Create(inbox));
+ return true;
}
catch (Exception ex)
{
@@ -35,7 +36,8 @@ namespace ZeroLevel.Network
{
try
{
- return _client.Send(FrameFactory.Create(inbox, data));
+ _client.Send(FrameFactory.Create(inbox, data));
+ return true;
}
catch (Exception ex)
{
@@ -47,8 +49,9 @@ namespace ZeroLevel.Network
public bool Send(T message)
{
try
- {
- return _client.Send(FrameFactory.Create(BaseSocket.DEFAULT_MESSAGE_INBOX, MessageSerializer.SerializeCompatible(message)));
+ {
+ _client.Send(FrameFactory.Create(BaseSocket.DEFAULT_MESSAGE_INBOX, MessageSerializer.SerializeCompatible(message)));
+ return true;
}
catch (Exception ex)
{
@@ -60,8 +63,9 @@ namespace ZeroLevel.Network
public bool Send(string inbox, T message)
{
try
- {
- return _client.Send(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible(message)));
+ {
+ _client.Send(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible(message)));
+ return true;
}
catch (Exception ex)
{
@@ -73,8 +77,9 @@ namespace ZeroLevel.Network
public bool Request(string inbox, Action callback)
{
try
- {
- return _client.Request(FrameFactory.Create(inbox), f => callback(f));
+ {
+ _client.Request(FrameFactory.Create(inbox), f => callback(f));
+ return true;
}
catch (Exception ex)
{
@@ -87,7 +92,8 @@ namespace ZeroLevel.Network
{
try
{
- return _client.Request(FrameFactory.Create(inbox, data), f => callback(f));
+ _client.Request(FrameFactory.Create(inbox, data), f => callback(f));
+ return true;
}
catch (Exception ex)
{
@@ -100,7 +106,8 @@ namespace ZeroLevel.Network
{
try
{
- return _client.Request(FrameFactory.Create(inbox), f => callback(MessageSerializer.DeserializeCompatible(f)));
+ _client.Request(FrameFactory.Create(inbox), f => callback(MessageSerializer.DeserializeCompatible(f)));
+ return true;
}
catch (Exception ex)
{
@@ -113,7 +120,8 @@ namespace ZeroLevel.Network
{
try
{
- return _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_INBOX), f => callback(MessageSerializer.DeserializeCompatible(f)));
+ _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_INBOX), f => callback(MessageSerializer.DeserializeCompatible(f)));
+ return true;
}
catch (Exception ex)
{
@@ -126,8 +134,9 @@ namespace ZeroLevel.Network
{
try
{
- return _client.Request(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible(request)),
+ _client.Request(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible(request)),
f => callback(MessageSerializer.DeserializeCompatible(f)));
+ return true;
}
catch (Exception ex)
{
@@ -140,8 +149,9 @@ namespace ZeroLevel.Network
{
try
{
- return _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_WITHOUT_ARGS_INBOX, MessageSerializer.SerializeCompatible(request)),
+ _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_WITHOUT_ARGS_INBOX, MessageSerializer.SerializeCompatible(request)),
f => callback(MessageSerializer.DeserializeCompatible(f)));
+ return true;
}
catch (Exception ex)
{
diff --git a/ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs b/ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs
index f64aec5..1fe8215 100644
--- a/ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs
+++ b/ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs
@@ -1,7 +1,6 @@
using MemoryPools;
using System;
using System.Threading;
-using ZeroLevel.Services.Pools;
namespace ZeroLevel.Network
{
diff --git a/ZeroLevel/Services/Network/SocketClient.cs b/ZeroLevel/Services/Network/SocketClient.cs
index 367eac9..d41e32c 100644
--- a/ZeroLevel/Services/Network/SocketClient.cs
+++ b/ZeroLevel/Services/Network/SocketClient.cs
@@ -1,4 +1,5 @@
-using System;
+using MemoryPools;
+using System;
using System.Collections.Concurrent;
using System.Net;
using System.Net.Sockets;
@@ -19,8 +20,17 @@ namespace ZeroLevel.Network
public int identity;
public byte[] data;
}
+ private struct OutcomingFrame
+ {
+ public bool is_request;
+ public int identity;
+ public byte[] data;
+ }
- private BlockingCollection _incoming_queue = new BlockingCollection();
+ private readonly JetValPool _outcomingFramesPool = new JetValPool();
+ private ConcurrentQueue _incoming_queue = new ConcurrentQueue();
+ private ConcurrentQueue _outcoming_queue = new ConcurrentQueue();
+ private ManualResetEventSlim _outcomingFrameEvent = new ManualResetEventSlim(false);
#endregion
private Socket _clientSocket;
@@ -31,6 +41,7 @@ namespace ZeroLevel.Network
private readonly object _reconnection_lock = new object();
private long _heartbeat_key;
private Thread _receiveThread;
+ private Thread _sendingThread;
#endregion Private
@@ -41,7 +52,6 @@ namespace ZeroLevel.Network
try
{
_clientSocket = new Socket(ep.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
- _clientSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontLinger, true);
_clientSocket.Connect(ep);
OnConnect(this);
}
@@ -81,6 +91,11 @@ namespace ZeroLevel.Network
_receiveThread.IsBackground = true;
_receiveThread.Start();
+ _sendingThread = new Thread(OutcomingFramesJob);
+ _sendingThread.IsBackground = true;
+ _sendingThread.Start();
+
+
_heartbeat_key = Sheduller.RemindEvery(TimeSpan.FromMilliseconds(MINIMUM_HEARTBEAT_UPDATE_PERIOD_MS), Heartbeat);
}
@@ -109,27 +124,26 @@ namespace ZeroLevel.Network
public event Action OnDisconnect = (_) => { };
public IPEndPoint Endpoint { get; }
- public bool Request(Frame frame, Action callback, Action fail = null)
+ public void Request(Frame frame, Action callback, Action fail = null)
{
- if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Request] Socket status: {Status}");
+ if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Request] Socket status: {Status}");
var data = NetworkPacketFactory.Reqeust(MessageSerializer.Serialize(frame), out int id);
_requests.RegisterForFrame(id, callback, fail);
- return Send(id, true, data);
+ Send(id, true, data);
}
- public bool Send(Frame frame)
+ public void Send(Frame frame)
{
if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Send] Socket status: {Status}");
var data = NetworkPacketFactory.Message(MessageSerializer.Serialize(frame));
- return Send(0, false, data);
+ Send(0, false, data);
}
- public bool Response(byte[] data, int identity)
+ public void Response(byte[] data, int identity)
{
if (data == null) throw new ArgumentNullException(nameof(data));
if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Response] Socket status: {Status}");
-
- return Send(0, false, NetworkPacketFactory.Response(data, identity));
+ Send(0, false, NetworkPacketFactory.Response(data, identity));
}
#endregion
@@ -140,7 +154,7 @@ namespace ZeroLevel.Network
try
{
if (type == FrameType.KeepAlive) return;
- _incoming_queue.Add(new IncomingFrame
+ _incoming_queue.Enqueue(new IncomingFrame
{
data = data,
type = type,
@@ -223,75 +237,94 @@ namespace ZeroLevel.Network
IncomingFrame frame = default(IncomingFrame);
while (Status != SocketClientStatus.Disposed)
{
- try
+ if (_incoming_queue.TryDequeue(out frame))
{
- if (_incoming_queue.TryTake(out frame, 100))
+ try
{
- try
+ switch (frame.type)
{
- switch (frame.type)
- {
- case FrameType.Message:
- Router?.HandleMessage(MessageSerializer.Deserialize(frame.data), this);
- break;
- case FrameType.Request:
+ case FrameType.Message:
+ Router?.HandleMessage(MessageSerializer.Deserialize(frame.data), this);
+ break;
+ case FrameType.Request:
+ {
+ Router?.HandleRequest(MessageSerializer.Deserialize(frame.data), this, frame.identity, (id, response) =>
{
- Router?.HandleRequest(MessageSerializer.Deserialize(frame.data), this, frame.identity, (id, response) =>
+ if (response != null)
{
- if (response != null)
- {
- this.Response(response, id);
- }
- });
- }
- break;
- case FrameType.Response:
- {
- _requests.Success(frame.identity, frame.data);
- }
- break;
- }
- }
- catch (Exception ex)
- {
- Log.SystemError(ex, "[SocketClient.IncomingFramesJob] Handle frame");
+ this.Response(response, id);
+ }
+ });
+ }
+ break;
+ case FrameType.Response:
+ {
+ _requests.Success(frame.identity, frame.data);
+ }
+ break;
}
}
- }
- catch (Exception ex)
- {
- Log.SystemError(ex, "[SocketClient.IncomingFramesJob] _incoming_queue.Take");
- if (Status != SocketClientStatus.Disposed)
+ catch (Exception ex)
{
- _incoming_queue.Dispose();
- _incoming_queue = new BlockingCollection();
+ Log.SystemError(ex, "[SocketClient.IncomingFramesJob] Handle frame");
}
- continue;
+ }
+ else
+ {
+ Thread.Sleep(100);
}
}
}
- private bool Send(int id, bool is_request, byte[] data)
+ private void OutcomingFramesJob()
{
- if (Status == SocketClientStatus.Working)
+ while (Status != SocketClientStatus.Disposed)
{
- try
+ if (Status == SocketClientStatus.Working)
{
- if (is_request)
- {
- _requests.StartSend(id);
+ if (_outcomingFrameEvent.Wait(100))
+ {
+ _outcomingFrameEvent.Reset();
}
- var sended = _clientSocket.Send(data, data.Length, SocketFlags.None);
- return sended == data.Length;
+ while (_outcoming_queue.TryDequeue(out var frame))
+ {
+ try
+ {
+ if (frame.is_request)
+ {
+ _requests.StartSend(frame.identity);
+ }
+ _clientSocket.Send(frame.data, frame.data.Length, SocketFlags.None);
+ //var sended = _clientSocket.Send(frame.data, frame.data.Length, SocketFlags.None);
+ //return sended == frame.data.Length;
+ }
+ catch (Exception ex)
+ {
+ Log.SystemError(ex, $"[SocketClient.OutcomingFramesJob] _str_clientSocketeam.Send");
+ Broken();
+ OnDisconnect(this);
+ }
+ finally
+ {
+ _outcomingFramesPool.Return(frame);
+ }
+ }
}
- catch (Exception ex)
+ else
{
- Log.SystemError(ex, $"[SocketClient.SendFramesJob] _str_clientSocketeam.Send");
- Broken();
- OnDisconnect(this);
+ Thread.Sleep(400);
}
}
- return false;
+ }
+
+ private void Send(int id, bool is_request, byte[] data)
+ {
+ var frame = _outcomingFramesPool.Get();
+ frame.data = data;
+ frame.identity = id;
+ frame.is_request = is_request;
+ _outcoming_queue.Enqueue(frame);
+ _outcomingFrameEvent.Set();
}
#endregion
diff --git a/ZeroLevel/Services/Semantic/LanguageDictionary.cs b/ZeroLevel/Services/Semantic/LanguageDictionary.cs
index f44f5fd..bb7ab0d 100644
--- a/ZeroLevel/Services/Semantic/LanguageDictionary.cs
+++ b/ZeroLevel/Services/Semantic/LanguageDictionary.cs
@@ -1,5 +1,4 @@
using System.Collections.Generic;
-using ZeroLevel.Services.Semantic;
using ZeroLevel.Services.Serialization;
namespace ZeroLevel.Services.Semantic
diff --git a/ZeroLevel/Services/Semantic/WordTokenizer.cs b/ZeroLevel/Services/Semantic/WordTokenizer.cs
index 9b6c0b4..a2bb5e9 100644
--- a/ZeroLevel/Services/Semantic/WordTokenizer.cs
+++ b/ZeroLevel/Services/Semantic/WordTokenizer.cs
@@ -44,5 +44,41 @@ namespace ZeroLevel.Services.Semantic
_pool.Return(buffer);
}
}
+
+ public static IEnumerable TokenizeCaseSensitive(string text)
+ {
+ int index = 0;
+ bool first = true;
+ var buffer = _pool.Rent(ARRAY_SIZE);
+ try
+ {
+ for (int i = 0; i < text?.Length; i++)
+ {
+ if (first && Char.IsLetter(text[i]))
+ {
+ first = false;
+ buffer[index++] = text[i];
+ }
+ else if (first == false && Char.IsLetterOrDigit(text[i]))
+ {
+ buffer[index++] = text[i];
+ }
+ else if (index > 0)
+ {
+ yield return new string(buffer, 0, index);
+ index = 0;
+ first = true;
+ }
+ }
+ if (index > 0)
+ {
+ yield return new string(buffer, 0, index);
+ }
+ }
+ finally
+ {
+ _pool.Return(buffer);
+ }
+ }
}
}
diff --git a/ZeroLevel/Services/Text/SuffixAutomata.cs b/ZeroLevel/Services/Text/SuffixAutomata.cs
new file mode 100644
index 0000000..80fd5c8
--- /dev/null
+++ b/ZeroLevel/Services/Text/SuffixAutomata.cs
@@ -0,0 +1,159 @@
+using System.Collections.Generic;
+using System.Linq;
+
+namespace ZeroLevel.Services.Text
+{
+ internal struct state
+ {
+ public int len;
+ public int link;
+ public Dictionary next;
+ }
+
+ public class SuffixAutomata
+ {
+ const int MAXLEN = 100000;
+ private state[] st = new state[MAXLEN * 2];
+ int sz, last;
+
+ public void Init(bool is_reused = false)
+ {
+ sz = last = 0;
+ st[0].len = 0;
+ st[0].link = -1;
+ ++sz;
+ // этот код нужен, только если автомат строится много раз для разных строк:
+ for (int i = 0; i < MAXLEN * 2; ++i)
+ {
+ st[i].next = new Dictionary();
+ }
+ }
+
+ public void Extend(char c)
+ {
+ int cur = sz++;
+ st[cur].len = st[last].len + 1;
+ int p;
+ for (p = last; p != -1 && !st[p].next.ContainsKey(c); p = st[p].link)
+ st[p].next[c] = cur;
+ if (p == -1)
+ st[cur].link = 0;
+ else
+ {
+ int q = st[p].next[c];
+ if (st[p].len + 1 == st[q].len)
+ st[cur].link = q;
+ else
+ {
+ int clone = sz++;
+ st[clone].len = st[p].len + 1;
+ st[clone].next = st[q].next;
+ st[clone].link = st[q].link;
+ for (; p != -1 && st[p].next.ContainsKey(c) && st[p].next[c] == q; p = st[p].link)
+ st[p].next[c] = clone;
+ st[q].link = st[cur].link = clone;
+ }
+ }
+ last = cur;
+ }
+
+ public bool IsSubstring(string w)
+ {
+ if (string.IsNullOrEmpty(w)) return true;
+ bool fail = false;
+ int n, si = 0;
+ for (; si < last; si++)
+ {
+ if (st[si].next.ContainsKey(w[0]))
+ {
+ var start = st[si];
+ for (int i = 0; i < w.Length; i++)
+ {
+ if (start.next.ContainsKey(w[i]) == false)
+ {
+ fail = true;
+ break;
+ }
+ n = start.next[w[i]];
+ start = st[n];
+ }
+ break;
+ }
+ }
+ if (si == last)
+ {
+ fail = true;
+ }
+ return (!fail);
+ }
+
+ public string Intersection(string t)
+ {
+ var entries = st.Where(x => x.next.ContainsKey(t[0])).ToArray();
+ var candidates = entries
+ .Select(s => Intersection(s, t));
+ if (candidates != null && candidates.Any())
+ {
+ var max = candidates.Max(s => s?.Length ?? 0);
+ return candidates.FirstOrDefault(c => c != null && c.Length == max);
+ }
+ return null;
+ /*
+ int v = 0, l = 0, best = 0, bestpos = 0;
+ for (int i = 0; i < (int)t.Length; ++i)
+ {
+ while (v > 0 && !st[v].next.ContainsKey(t[i]))
+ {
+ v = st[v].link;
+ l = st[v].len;
+ }
+ if (st[v].next.ContainsKey(t[i]))
+ {
+ v = st[v].next[t[i]];
+ ++l;
+ }
+ if (l > best)
+ {
+ best = l;
+ bestpos = i;
+ }
+ }
+ var start = bestpos - best + 1;
+ var length = best;
+ if (start >= 0 && start < t.Length && (start + length) <= t.Length)
+ return t.Substring(start, length);
+ return null;
+ */
+ }
+
+ private string Intersection(state entry, string t)
+ {
+ int v = 0, l = 0, best = 0, bestpos = 0;
+ for (int i = 0; i < (int)t.Length; ++i)
+ {
+ while (v > 0 && !entry.next.ContainsKey(t[i]))
+ {
+ v = entry.link;
+ l = entry.len;
+ entry = st[v];
+ }
+ if (entry.next.ContainsKey(t[i]))
+ {
+ v = entry.next[t[i]];
+ entry = st[v];
+ ++l;
+ }
+ if (l > best)
+ {
+ best = l;
+ bestpos = i;
+ }
+ }
+ var start = bestpos - best + 1;
+ var length = best;
+ if (start >= 0 && start < t.Length && (start + length) <= t.Length)
+ return t.Substring(start, length);
+ return null;
+ }
+ }
+}
diff --git a/ZeroLevel/Services/Text/TStringReader.cs b/ZeroLevel/Services/Text/TStringReader.cs
index 4f5cce4..4875c02 100644
--- a/ZeroLevel/Services/Text/TStringReader.cs
+++ b/ZeroLevel/Services/Text/TStringReader.cs
@@ -39,9 +39,13 @@
public void SkipBreaks()
{
- while (EOF == false && char.IsWhiteSpace(Current)) Move();
+ while (EOF == false && (Current == '\n' || Current == '\r')) Move();
}
+ public bool IsBrackets() => EOF == false && (Current == '\'' || Current == '"' || Current == '”');
+ public bool IsBreak() => EOF == false && (Current == '\n' || Current == '\r');
+ public bool IsWhiteSpace() => EOF == false && char.IsWhiteSpace(Current);
+
public bool MoveBack()
{
_position = _position - 1;
@@ -84,7 +88,7 @@
index++;
identity = _template.Substring(offset, index - offset);
}
- return identity.ToLowerInvariant();
+ return identity;
}
public string ReadWord()
diff --git a/ZeroLevel/Services/Transliteration.cs b/ZeroLevel/Services/Transliteration.cs
index f24c5a9..1a15def 100644
--- a/ZeroLevel/Services/Transliteration.cs
+++ b/ZeroLevel/Services/Transliteration.cs
@@ -22,7 +22,7 @@ namespace ZeroLevel.Services
output = Regex.Replace(output, @"\s|\.|\(", " ");
output = Regex.Replace(output, @"\s+", " ");
- output = Regex.Replace(output, @"[^\s\w\d-]", "");
+ //output = Regex.Replace(output, @"[^\s\w\d-]", "");
output = output.Trim();
Dictionary tdict = GetDictionaryByType(type);
@@ -145,7 +145,7 @@ namespace ZeroLevel.Services
gost.Add("«", "");
gost.Add("»", "");
gost.Add("—", "-");
- gost.Add(" ", "-");
+ gost.Add(" ", " ");
iso.Add("Є", "YE");
iso.Add("І", "I");
@@ -223,7 +223,7 @@ namespace ZeroLevel.Services
iso.Add("«", "");
iso.Add("»", "");
iso.Add("—", "-");
- iso.Add(" ", "-");
+ iso.Add(" ", " ");
}
}
}
diff --git a/ZeroLevel/Services/Web/HtmlUtility.cs b/ZeroLevel/Services/Web/HtmlUtility.cs
index ab1b148..5c5318f 100644
--- a/ZeroLevel/Services/Web/HtmlUtility.cs
+++ b/ZeroLevel/Services/Web/HtmlUtility.cs
@@ -346,7 +346,7 @@ namespace ZeroLevel.Services.Web
if (cursor.Current == '&')
{
cursor.Move();
- var identity = cursor.ReadIdentity();
+ var identity = cursor.ReadIdentity().ToLowerInvariant();
cursor.MoveBack();
if (_entityMap.ContainsKey(identity + ";"))
{
diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj
index 8ca8510..9122ef0 100644
--- a/ZeroLevel/ZeroLevel.csproj
+++ b/ZeroLevel/ZeroLevel.csproj
@@ -6,7 +6,7 @@
ogoun
ogoun
- 3.3.6.3
+ 3.3.6.4
Configuration floating number convert fix
https://github.com/ogoun/Zero/wiki
Copyright Ogoun 2022
@@ -14,8 +14,8 @@
https://github.com/ogoun/Zero
git
- 3.3.6.3
- 3.3.6.3
+ 3.3.6.4
+ 3.3.6.4
AnyCPU;x64;x86
zero.png
full