Improve quality

pull/3/head
Ogoun 2 years ago
parent b82fa90033
commit b95c1274eb

1
.gitignore vendored

@ -30,6 +30,7 @@ bower_components
*.nupkg
*.p7s
*.sln.ide
*.onnx
[Bb]in
[Dd]ebug*/
[Oo]bj/

@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>

@ -0,0 +1,50 @@
namespace AppContainer
{
public class Endpoint
{
public string VersionEndpoint { get; set; }
public string AppEndpoint { get; set; }
public string PingEndpoint { get; set; }
public string Token { get; set; }
}
public class HostSettings
{
public IEnumerable<Endpoint> Endpoints { get; set; }
public string EntryClass { get; set; }
public string EntryMethod { get; set; }
}
public class Host
{
private readonly HostSettings _settings;
public Host(HostSettings settings)
{
if (settings == null) throw new ArgumentNullException(nameof(settings));
_settings = settings;
}
public async Task<bool> HasUpdate()
{
if (_settings.Endpoints != null)
{
foreach (var ep in _settings.Endpoints)
{
try
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("token", ep.Token);
var response = await client.GetAsync(ep.PingEndpoint);
response.EnsureSuccessStatusCode();
}
}
catch (Exception ex)
{
}
}
}
}
}
}

@ -1,9 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<Platforms>AnyCPU;x64;x86</Platforms>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>

@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net;
using ZeroLevel;
using ZeroLevel.Network;
using ZeroLevel.Services.HashFunctions;
@ -60,6 +57,9 @@ namespace Client
{
private readonly static XXHashUnsafe _hash = new XXHashUnsafe(667);
private static long _successSend = 0;
private static long _faultSend = 0;
static void Main(string[] args)
{
Log.AddConsoleLogger();
@ -68,6 +68,12 @@ namespace Client
var port = ReadPort();
ex.RoutesStorage.Set("server", new IPEndPoint(address, port));
Sheduller.RemindEvery(TimeSpan.FromMilliseconds(300), () =>
{
Console.SetCursorPosition(0, 0);
Console.WriteLine($"Success/Fault network: {_successSend} / {_faultSend}");
});
uint index = 0;
while (true)
{
@ -99,7 +105,7 @@ namespace Client
var info = new Info { Checksum = full_checksum, Id = id, Length = length };
if (client.Request<Info, bool>("start", info, res =>
{
Log.Info($"Success start sending packet '{id}'");
Interlocked.Increment(ref _successSend);
}))
{
uint size = 1;
@ -111,11 +117,15 @@ namespace Client
{
if (!res)
{
Log.Info($"Fault server incoming packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes.");
Interlocked.Increment(ref _faultSend);
}
else
{
Interlocked.Increment(ref _successSend);
}
}))
{
Log.Warning($"Can't start send packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes. No connection");
Interlocked.Increment(ref _faultSend);
}
offset += size;
size += 1;
@ -124,7 +134,7 @@ namespace Client
}
else
{
Log.Warning($"Can't start send packet '{id}'. No connection");
Interlocked.Increment(ref _faultSend);
}
}
@ -135,7 +145,7 @@ namespace Client
var info = new Info { Checksum = full_checksum, Id = id, Length = length };
if (exchange.Request<Info, bool>("server", "start", info))
{
Log.Info($"Success start sending packet '{id}'");
Interlocked.Increment(ref _successSend);
uint size = 4096;
uint offset = 0;
while (offset < payload.Length)
@ -143,7 +153,11 @@ namespace Client
var fragment = GetFragment(id, payload, offset, size);
if (!exchange.Request<Fragment, bool>("server", "part", fragment))
{
Log.Info($"Fault server incoming packet '{id}' fragment. Offset: '{offset}'. Size: '{size}' bytes.");
Interlocked.Increment(ref _faultSend);
}
else
{
Interlocked.Increment(ref _successSend);
}
offset += size;
}
@ -151,7 +165,7 @@ namespace Client
}
else
{
Log.Warning($"Can't start send packet '{id}'. No connection");
Interlocked.Increment(ref _faultSend);
}
}

@ -1,13 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
https://go.microsoft.com/fwlink/?LinkID=208121.
-->
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<PublishProtocol>FileSystem</PublishProtocol>
<Configuration>Release</Configuration>
<Platform>Any CPU</Platform>
<TargetFramework>netcoreapp3.1</TargetFramework>
<PublishDir>bin\Release\netcoreapp3.1\publish\</PublishDir>
</PropertyGroup>
</Project>

@ -6,6 +6,7 @@ using ZeroLevel.Services.Serialization;
namespace Server
{
//netsh advfirewall firewall add rule name="ClientTest" dir=in action=allow protocol=TCP localport=5016
public class Info
: IBinarySerializable
{

@ -1,28 +0,0 @@
using System.Threading;
namespace TestApp
{
public class HybrydLock
{
private AutoResetEvent _lock = new AutoResetEvent(false);
private volatile int _counter = 0;
public void Enter()
{
if (Interlocked.Increment(ref _counter) == 1)
{
return;
}
_lock.WaitOne();
}
public void Leave()
{
if (Interlocked.Decrement(ref _counter) == 0)
{
return;
}
_lock.Set();
}
}
}

@ -1,72 +0,0 @@
using System;
using System.Net;
using System.Threading;
using ZeroLevel;
using ZeroLevel.Network;
using ZeroLevel.Services.Applications;
namespace TestApp
{
public class MyService
: BaseZeroService
{
public MyService()
: base()
{
}
protected override void StartAction()
{
Log.Info("Started");
ReadServiceInfo();
var host = UseHost(8800);
AutoregisterInboxes(host);
host.OnConnect += Host_OnConnect;
host.OnDisconnect += Host_OnDisconnect;
int counter = 0;
Sheduller.RemindEvery(TimeSpan.FromSeconds(1), () =>
{
Log.Info($"RPS: {counter}");
Interlocked.Exchange(ref counter, 0);
});
Exchange.RoutesStorage.Set("test.app", new IPEndPoint(IPAddress.Loopback, 8800));
while (true)
{
try
{
//var s = Exchange.Request<int>("test.app", "counter");
Interlocked.Add(ref counter, Exchange.Request<int>("test.app", "counter"));
}
catch(Exception ex)
{
Log.Error(ex, "Request fault");
Thread.Sleep(300);
}
}
}
private void Host_OnDisconnect(ISocketClient obj)
{
Log.Info($"Client '{obj.Endpoint.Address}:{obj.Endpoint.Port}' disconnected");
}
private void Host_OnConnect(IClient obj)
{
Log.Info($"Client '{obj.Socket.Endpoint.Address}:{obj.Socket.Endpoint.Port}' connected");
}
[ExchangeReplierWithoutArg("counter")]
public int GetCounter(ISocketClient client)
{
return 1;
}
protected override void StopAction()
{
Log.Info("Stopped");
}
}
}

@ -0,0 +1,44 @@
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using System.Collections.Generic;
using System.Linq;
using ZeroLevel.NN.Architectures.YoloV5;
using ZeroLevel.NN.Models;
using ZeroLevel.NN.Services;
namespace TestApp
{
public class PersonDetector
{
private const string MODEL_PATH = @"nnmodels/Yolo5S/yolov5s327e.onnx";
private readonly Yolov5Detector _detector;
private float _threshold = 0.17f;
public PersonDetector()
{
_detector = new Yolov5Detector(MODEL_PATH, gpu: false);
}
public IEnumerable<YoloPrediction> Detect(string imagePath)
{
using (Image<Rgb24> image = Image.Load<Rgb24>(imagePath))
{
var t_predictions = _detector.PredictMultiply(image, true, _threshold);
t_predictions.Apply(p =>
{
p.Cx /= image.Width;
p.Cy /= image.Height;
});
if (t_predictions != null)
{
t_predictions.RemoveAll(p => (p.W * image.Width) < 10.0f || (p.H * image.Height) < 10.0f);
}
if (t_predictions.Count > 0)
{
NMS.Apply(t_predictions);
return t_predictions;
}
}
return Enumerable.Empty<YoloPrediction>();
}
}
}

@ -1,84 +1,11 @@
using Newtonsoft.Json;
using System;
using System.Net;
using System.Threading;
using ZeroLevel;
using ZeroLevel.Logging;
using ZeroLevel.Network;
using ZeroLevel.Services.Serialization;
using ZeroLevel.Services.Trees;
namespace TestApp
namespace TestApp
{
public class TestQuery
{
public string Name { get; set; }
public int Age { get; set; }
public string[] Roles { get; set; }
}
internal static class Program
{
private static string Serialize(object instance)
{
return JsonConvert.SerializeObject(instance);
}
private static void Main(string[] args)
{
Configuration.Save(Configuration.ReadFromApplicationConfig());
Bootstrap.Startup<MyService>(args,
() => Configuration.ReadSetFromIniFile("config.ini"))
.EnableConsoleLog(LogLevel.System | LogLevel.FullDebug)
//.UseDiscovery()
.Run()
.WaitWhileStatus(ZeroServiceStatus.Running)
.Stop();
Bootstrap.Shutdown();
}
static void SimpleCSTest()
{
var server_router = new Router();
server_router.RegisterInbox<string>("test", (c, line) =>
{
Console.WriteLine(line);
});
server_router.RegisterInbox<string, string>("req", (c, line) =>
{
Console.WriteLine($"Request: {line}");
return line.ToUpperInvariant();
});
var server = new SocketServer(new System.Net.IPEndPoint(IPAddress.Any, 666), server_router);
var client_router = new Router();
var client = new SocketClient(new IPEndPoint(IPAddress.Loopback, 666), client_router);
var frm = FrameFactory.Create("req", MessageSerializer.SerializeCompatible("Hello world"));
while (Console.KeyAvailable == false)
{
client.Request(frm, data =>
{
var line = MessageSerializer.DeserializeCompatible<string>(data);
Console.WriteLine($"Response: {line}");
});
Thread.Sleep(2000);
}
}
public static double[] Generate(int vector_size)
{
var rnd = new Random((int)Environment.TickCount);
var vector = new double[vector_size];
for (int i = 0; i < vector_size; i++)
{
vector[i] = 50.0d - rnd.NextDouble() * 100.0d;
}
return vector;
var detector = new PersonDetector();
var predictions = detector.Detect(@"E:\Desktop\test\1.JPG");
}
}
}

@ -11,6 +11,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ZeroLevel.NN\ZeroLevel.NN.csproj" />
<ProjectReference Include="..\ZeroLevel\ZeroLevel.csproj" />
</ItemGroup>
@ -18,6 +19,9 @@
<None Update="config.ini">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="nnmodels\Yolo5S\yolov5s327e.onnx">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

@ -0,0 +1,37 @@
using Microsoft.ML.OnnxRuntime.Tensors;
using SixLabors.ImageSharp;
using ZeroLevel.NN.Models;
namespace ZeroLevel.NN
{
public class GenderAgeEstimator
: SSDNN
{
private const int INPUT_WIDTH = 64;
private const int INPUT_HEIGHT = 64;
public GenderAgeEstimator(string modelPath, bool gpu = false) : base(modelPath, gpu)
{
}
public (Gender, int) Predict(Image image)
{
var input = MakeInput(image,
new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst)
.ApplyNormilization()
.ApplyAxeInversion());
return Predict(input);
}
public (Gender, int) Predict(Tensor<float> input)
{
float[] variances = null;
Extract(new Dictionary<string, Tensor<float>> { { "input", input } }, d =>
{
variances = d.First().Value.ToArray();
});
var gender = Argmax(variances[0..2]) == 0 ? Gender.Male : Gender.Feemale;
return (gender, (int)variances[2]);
}
}
}

@ -19,7 +19,7 @@ namespace ZeroLevel.NN
/// <summary>
/// Input tensor is 1 x 3 x height x width with mean values 104, 117, 123. Input image have to be previously resized to 224 x 224 pixels and converted to BGR format.
/// </summary>
public class GoogleAgeDetector
public class GoogleAgeEstimator
: SSDNN
{
private const int INPUT_WIDTH = 224;
@ -28,7 +28,7 @@ namespace ZeroLevel.NN
private Age[] _ageList = new[] { Age.From0To2, Age.From4To6, Age.From8To12, Age.From15To20, Age.From25To32, Age.From38To43, Age.From48To53, Age.From60To100 };
public GoogleAgeDetector(string modelPath, bool gpu = false) : base(modelPath, gpu)
public GoogleAgeEstimator(string modelPath, bool gpu = false) : base(modelPath, gpu)
{
}
@ -48,7 +48,7 @@ namespace ZeroLevel.NN
{
variances = d.First().Value.ToArray();
});
var (number, index) = variances.Select((n, i) => (n, i)).Max();
var index = Argmax(variances);
return _ageList[index];
}
}

@ -1,6 +1,7 @@
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using ZeroLevel;
using ZeroLevel.NN;
using ZeroLevel.NN.Models;
@ -127,25 +128,36 @@ namespace Zero.NN.Services
var around_x2 = centerFaceX + radius;
var around_y1 = centerFaceY - radius;
var around_y2 = centerFaceY + radius;
using (var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2))
try
{
var matrix = Face.GetTransformMatrix(face);
var builder = new AffineTransformBuilder();
builder.AppendMatrix(matrix);
faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic));
vector = _encoder.Predict(faceImage);
/*var aligned_faces = detector.Predict(faceImage);
if (aligned_faces != null && aligned_faces.Count == 1)
using (var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2))
{
using (var ci = SpecialCrop(faceImage, aligned_faces[0]))
var matrix = Face.GetTransformMatrix(face);
var builder = new AffineTransformBuilder();
builder.AppendMatrix(matrix);
faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic));
vector = _encoder.Predict(faceImage);
/*var aligned_faces = detector.Predict(faceImage);
if (aligned_faces != null && aligned_faces.Count == 1)
{
vector = encoder.Predict(faceImage);
using (var ci = SpecialCrop(faceImage, aligned_faces[0]))
{
vector = encoder.Predict(faceImage);
}
}
else
{
vector = encoder.Predict(faceImage);
}*/
}
else
}
catch (Exception ex)
{
Log.SystemError(ex, "[FaceSeacrhService.GetEmbeddings]");
using (var faceImage = ImagePreprocessor.Crop(image, face.X1, face.Y1, face.X2, face.Y2))
{
vector = encoder.Predict(faceImage);
}*/
vector = _encoder.Predict(faceImage);
}
}
}
else
@ -162,5 +174,53 @@ namespace Zero.NN.Services
};
}
}
public IEnumerable<(FaceEmbedding, Image)> GetEmbeddingsAndCrop(Image<Rgb24> image)
{
int width = image.Width;
int heigth = image.Height;
var faces = _detector.Predict(image);
foreach (var face in faces)
{
Face.FixInScreen(face, width, heigth);
float[] vector;
if (_useFaceAlign)
{
int x = (int)face.X1;
int y = (int)face.Y1;
int w = (int)(face.X2 - face.X1);
int h = (int)(face.Y2 - face.Y1);
var radius = (float)Math.Sqrt(w * w + h * h) / 2f;
var centerFaceX = (face.X2 + face.X1) / 2.0f;
var centerFaceY = (face.Y2 + face.Y1) / 2.0f;
var around_x1 = centerFaceX - radius;
var around_x2 = centerFaceX + radius;
var around_y1 = centerFaceY - radius;
var around_y2 = centerFaceY + radius;
var faceImage = ImagePreprocessor.Crop(image, around_x1, around_y1, around_x2, around_y2);
var matrix = Face.GetTransformMatrix(face);
var builder = new AffineTransformBuilder();
builder.AppendMatrix(matrix);
faceImage.Mutate(x => x.Transform(builder, KnownResamplers.Bicubic));
vector = _encoder.Predict(faceImage);
yield return (new FaceEmbedding
{
Face = face,
Vector = vector
}, faceImage);
}
else
{
var faceImage = ImagePreprocessor.Crop(image, face.X1, face.Y1, face.X2, face.Y2);
vector = _encoder.Predict(faceImage);
yield return (new FaceEmbedding
{
Face = face,
Vector = vector
}, faceImage);
}
}
}
}
}

@ -0,0 +1,144 @@
using Microsoft.ML.OnnxRuntime.Tensors;
using SixLabors.ImageSharp;
using ZeroLevel.NN.Models;
namespace ZeroLevel.NN.Architectures.YoloV5
{
public class Yolov5Detector
: SSDNN
{
private int INPUT_WIDTH = 640;
private int INPUT_HEIGHT = 640;
private int CROP_WIDTH = 1440;
private int CROP_HEIGHT = 1440;
public Yolov5Detector(string modelPath, int inputWidth = 640, int inputHeight = 640, bool gpu = false)
: base(modelPath, gpu)
{
INPUT_HEIGHT = inputHeight;
INPUT_WIDTH = inputWidth;
}
public List<YoloPrediction> Predict(Image image, float threshold)
{
var input = MakeInput(image,
new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst)
.ApplyNormilization()
.ApplyAxeInversion());
return Predict(input, threshold);
}
public List<YoloPrediction> PredictMultiply(Image image, bool withFullResizedImage, float threshold)
{
var input = MakeInputBatch(image,
new ImagePreprocessorOptions(INPUT_WIDTH, INPUT_HEIGHT, PredictorChannelType.ChannelFirst)
.ApplyNormilization()
.ApplyAxeInversion()
.UseCrop(CROP_WIDTH, CROP_HEIGHT, withFullResizedImage, true));
return PredictMultiply(input, threshold);
}
public List<YoloPrediction> Predict(Tensor<float> input, float threshold)
{
var result = new List<YoloPrediction>();
Extract(new Dictionary<string, Tensor<float>> { { "images", input } }, d =>
{
var output = d["output"];
/*
var output350 = d["350"];
var output498 = d["498"];
var output646 = d["646"];
*/
if (output != null && output != null)
{
var relative_koef_x = 1.0f / INPUT_WIDTH;
var relative_koef_y = 1.0f / INPUT_HEIGHT;
for (int box = 0; box < output.Dimensions[1]; box++)
{
var conf = output[0, box, 4]; // уверенность в наличии любого объекта
if (conf > threshold)
{
var class_confidense = output[0, box, 5]; // уверенность в наличии объекта класса person
if (class_confidense > threshold)
{
// Перевод относительно входа модели в относительные координаты
var cx = output[0, box, 0] * relative_koef_x;
var cy = output[0, box, 1] * relative_koef_y;
var h = output[0, box, 2] * relative_koef_y;
var w = output[0, box, 3] * relative_koef_x;
result.Add(new YoloPrediction
{
Cx = cx,
Cy = cy,
W = w,
H = h,
Class = 0,
Label = "0",
Score = conf
});
}
}
}
}
});
return result;
}
public List<YoloPrediction> PredictMultiply(ImagePredictionInput[] inputs, float threshold)
{
var result = new List<YoloPrediction>();
var relative_koef_x = 1.0f / (float)INPUT_WIDTH;
var relative_koef_y = 1.0f / (float)INPUT_HEIGHT;
foreach (var input in inputs)
{
Extract(new Dictionary<string, Tensor<float>> { { "images", input.Tensor } }, d =>
{
var output = d["output"];
/*
var output350 = d["350"];
var output498 = d["498"];
var output646 = d["646"];
*/
if (output != null && output != null)
{
for (int index = 0; index < input.Count; index++)
{
var real_koef_x = (float)input.Offsets[index].Width / (float)INPUT_WIDTH;
var real_koef_y = (float)input.Offsets[index].Height / (float)INPUT_HEIGHT;
for (int box = 0; box < output.Dimensions[1]; box++)
{
var conf = output[index, box, 4]; // уверенность в наличии любого объекта
if (conf > threshold)
{
var class_confidense = output[index, box, 5]; // уверенность в наличии объекта класса person
if (class_confidense > threshold)
{
// Перевод относительно входа модели в относительные координаты
var cx = output[index, box, 0] * real_koef_x;
var cy = output[index, box, 1] * real_koef_y;
var h = output[index, box, 2] * relative_koef_y;
var w = output[index, box, 3] * relative_koef_x;
// Перевод в координаты отнисительно текущего смещения
cx += input.Offsets[index].X;
cy += input.Offsets[index].Y;
result.Add(new YoloPrediction
{
Cx = cx,
Cy = cy,
W = w,
H = h,
Class = 0,
Label = "0",
Score = conf
});
}
}
}
}
}
});
}
return result;
}
}
}

@ -0,0 +1,9 @@
namespace ZeroLevel.NN.Models
{
public enum Gender
{
Unknown = 0,
Male = 1,
Feemale = 2
}
}

@ -2,7 +2,7 @@
namespace ZeroLevel.NN.Models
{
public class PredictionInput
public class ImagePredictionInput
{
public Tensor<float> Tensor;
public OffsetBox[] Offsets;

@ -2,7 +2,7 @@
{
public class ImagePreprocessorOptions
{
private const float PIXEL_NORMALIZATION_SCALE = 1f / 255f;
private const float PIXEL_NORMALIZATION_SCALE = 1.0f / 255.0f;
public ImagePreprocessorOptions(int inputWidth, int inputHeight, PredictorChannelType channelType)
{
this.InputWidth = inputWidth;

@ -0,0 +1,66 @@
using ZeroLevel.Services.Serialization;
namespace ZeroLevel.NN.Models
{
public class YoloPrediction
: IBinarySerializable
{
public int Class { get; set; }
public float Cx { get; set; }
public float Cy { get; set; }
public float W { get; set; }
public float H { get; set; }
public float Score { get; set; }
public string Label { get; set; }
public float X { get { return Cx - W / 2.0f; } }
public float Y { get { return Cy - W / 2.0f; } }
public float Area { get { return W * H; } }
public string Description
{
get
{
return $"{Label} ({(int)(Score * 100)} %)";
}
}
public float this[int index]
{
get
{
switch (index)
{
case 0: return Cx;
case 1: return Cy;
case 2: return Cx + W;
case 3: return Cy + H;
}
return 0;
}
}
public void Deserialize(IBinaryReader reader)
{
this.Cx = reader.ReadFloat();
this.Cy = reader.ReadFloat();
this.W = reader.ReadFloat();
this.H = reader.ReadFloat();
this.Class = reader.ReadInt32();
this.Score = reader.ReadFloat();
this.Label = reader.ReadString();
}
public void Serialize(IBinaryWriter writer)
{
writer.WriteFloat(this.Cx);
writer.WriteFloat(this.Cy);
writer.WriteFloat(this.W);
writer.WriteFloat(this.H);
writer.WriteInt32(this.Class);
writer.WriteFloat(this.Score);
writer.WriteString(this.Label);
}
}
}

@ -8,6 +8,8 @@ namespace ZeroLevel.NN
{
public static class ImagePreprocessor
{
private static Action<Tensor<float>, float, int, int, int, int> _precompiledChannelFirstAction = new Action<Tensor<float>, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, c, i, j] = v; });
private static Action<Tensor<float>, float, int, int, int, int> _precompiledChannelLastAction = new Action<Tensor<float>, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, i, j, c] = v; });
private static Func<byte, int, float> PixelToTensorMethod(ImagePreprocessorOptions options)
{
if (options.Normalize)
@ -42,26 +44,51 @@ namespace ZeroLevel.NN
return new Func<byte, int, float>((b, _) => (float)b);
}
//private static int CalculateFragmentsCount(Image image, ImagePreprocessorOptions options)
//{
// int count = (options.Crop.SaveOriginal ? 1 : 0);
// var Sw = image.Width; // ширина оригинала
// var Sh = image.Height; // высота оригинала
// var CRw = options.Crop.Width; // ширина кропа
// var CRh = options.Crop.Height; // высота кропа
// var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw; // сдвиг по OX к следующему кропу
// var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh; // сдвиг по OY к следующему кропу
// for (int x = 0; x < Sw; x += Dx)
// {
// for (int y = 0; y < Sh; y += Dy)
// {
// count++;
// }
// }
// return count;
//}
private static int CalculateFragmentsCount(Image image, ImagePreprocessorOptions options)
{
int count = 0;
var xs = options.Crop.Overlap ? (int)(options.Crop.Width * options.Crop.OverlapKoefWidth) : options.Crop.Width;
var ys = options.Crop.Overlap ? (int)(options.Crop.Height * options.Crop.OverlapKoefHeight) : options.Crop.Height;
for (var x = 0; x < image.Width - xs; x += xs)
int count = (options.Crop.SaveOriginal ? 1 : 0);
var Sw = image.Width; // ширина оригинала
var Sh = image.Height; // высота оригинала
var CRw = options.InputWidth; // ширина кропа (равна ширине входа, т.к. изображение отресайзено подобающим образом)
var CRh = options.InputHeight; // высота кропа (равна высоте входа, т.к. изображение отресайзено подобающим образом)
var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw; // сдвиг по OX к следующему кропу
var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh; // сдвиг по OY к следующему кропу
for (int x = 0; x < Sw; x += Dx)
{
for (var y = 0; y < image.Height - ys; y += ys)
for (int y = 0; y < Sh; y += Dy)
{
count++;
}
}
return count;
}
private static void FillTensor(Tensor<float> tensor, Image image, int index, ImagePreprocessorOptions options, Func<byte, int, float> pixToTensor)
{
var append = options.ChannelType == PredictorChannelType.ChannelFirst
? new Action<Tensor<float>, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, c, i, j] = v; })
: new Action<Tensor<float>, float, int, int, int, int>((t, v, ind, c, i, j) => { t[ind, i, j, c] = v; });
var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction;
((Image<Rgb24>)image).ProcessPixelRows(pixels =>
{
@ -112,6 +139,59 @@ namespace ZeroLevel.NN
});
}
private static void FillTensor(Tensor<float> tensor, Image image, int startX, int startY, int w, int h, int index, ImagePreprocessorOptions options, Func<byte, int, float> pixToTensor)
{
var append = options.ChannelType == PredictorChannelType.ChannelFirst ? _precompiledChannelFirstAction : _precompiledChannelLastAction;
((Image<Rgb24>)image).ProcessPixelRows(pixels =>
{
if (options.InvertXY)
{
for (int y = startY; y < h; y++)
{
Span<Rgb24> pixelSpan = pixels.GetRowSpan(y);
for (int x = startX; x < w; x++)
{
if (options.BGR)
{
append(tensor, pixToTensor(pixelSpan[x].B, 0), index, 0, y, x);
append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, y, x);
append(tensor, pixToTensor(pixelSpan[x].R, 2), index, 2, y, x);
}
else
{
append(tensor, pixToTensor(pixelSpan[x].R, 0), index, 0, y, x);
append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, y, x);
append(tensor, pixToTensor(pixelSpan[x].B, 2), index, 2, y, x);
}
}
}
}
else
{
for (int y = startY; y < h; y++)
{
Span<Rgb24> pixelSpan = pixels.GetRowSpan(y);
for (int x = startX; x < w; x++)
{
if (options.BGR)
{
append(tensor, pixToTensor(pixelSpan[x].B, 0), index, 0, x, y);
append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, x, y);
append(tensor, pixToTensor(pixelSpan[x].R, 2), index, 2, x, y);
}
else
{
append(tensor, pixToTensor(pixelSpan[x].R, 0), index, 0, x, y);
append(tensor, pixToTensor(pixelSpan[x].G, 1), index, 1, x, y);
append(tensor, pixToTensor(pixelSpan[x].B, 2), index, 2, x, y);
}
}
}
}
});
}
private static Tensor<float> InitInputTensor(ImagePreprocessorOptions options, int batchSize = 1)
{
switch (options.ChannelType)
@ -127,101 +207,139 @@ namespace ZeroLevel.NN
}
}
public static PredictionInput[] ToTensors(this Image image, ImagePreprocessorOptions options)
public static ImagePredictionInput[] ToTensors(this Image image, ImagePreprocessorOptions options)
{
PredictionInput[] result = null;
ImagePredictionInput[] result = null;
var pixToTensor = PixelToTensorMethod(options);
options.Channels = image.PixelType.BitsPerPixel >> 3;
if (options.Crop.Enabled)
{
var fragments = CalculateFragmentsCount(image, options);
int count = CalculateFragmentsCount(image, options) + (options.Crop.SaveOriginal ? 1 : 0);
int offset = count % options.MaxBatchSize;
int count_tensors = count / options.MaxBatchSize + (offset == 0 ? 0 : 1);
var tensors = new PredictionInput[count_tensors];
for (int i = 0; i < count_tensors; i++)
{
if (i < count_tensors - 1)
{
tensors[i] = new PredictionInput
{
Tensor = InitInputTensor(options, options.MaxBatchSize),
Offsets = new OffsetBox[options.MaxBatchSize],
Count = options.MaxBatchSize
};
}
else
{
tensors[i] = new PredictionInput
{
Tensor = InitInputTensor(options, offset == 0 ? options.MaxBatchSize : offset),
Offsets = new OffsetBox[offset == 0 ? options.MaxBatchSize : offset],
Count = offset == 0 ? options.MaxBatchSize : offset
};
}
}
// Размеры оригинального изображения
var Sw = image.Width;
var Sh = image.Height;
int tensor_index = 0;
int tensor_part_index = 0;
var xs = options.Crop.Overlap ? (int)(options.Crop.Width * options.Crop.OverlapKoefWidth) : options.Crop.Width;
var ys = options.Crop.Overlap ? (int)(options.Crop.Height * options.Crop.OverlapKoefHeight) : options.Crop.Height;
// Создание ресайза для целочисленного прохода кропами шириной CRw и высотой CRh
var resizedForCropWidthKoef = options.InputWidth / (double)options.Crop.Width;
var resizedForCropHeightKoef = options.InputHeight / (double)options.Crop.Height;
if (options.Crop.SaveOriginal)
{
using (var copy = image.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
{
FillTensor(tensors[tensor_index].Tensor, copy, tensor_part_index, options, pixToTensor);
tensors[tensor_index].Offsets[tensor_part_index] = new OffsetBox(0, 0, image.Width, image.Height);
}
tensor_part_index++;
}
for (var x = 0; x < image.Width - xs; x += xs)
// Размеры для ресайза изображения к размеру по которому удобно идти кропами
var resizedForCropWidth = (int)Math.Round(Sw * resizedForCropWidthKoef, MidpointRounding.ToEven);
var resizedForCropHeight = (int)Math.Round(Sh * resizedForCropHeightKoef, MidpointRounding.ToEven);
// Размеры кропа, равны входу сети, а не (options.Crop.Width, options.Crop.Height), т.к. для оптимизации изображение будет предварительно отресайзено
var CRw = options.InputWidth;
var CRh = options.InputHeight;
// Расчет сдвигов между кропами
var Dx = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefWidth * CRw) : CRw;
var Dy = options.Crop.Overlap ? (int)(options.Crop.OverlapKoefHeight * CRh) : CRh;
using (var source = image.Clone(img => img.Resize(resizedForCropWidth, resizedForCropHeight, KnownResamplers.Bicubic)))
{
var startx = x;
var dx = (x + options.Crop.Width) - image.Width;
if (dx > 0)
{
startx -= dx;
}
for (var y = 0; y < image.Height - ys; y += ys)
// Количество тензоров всего, во всех батчах суммарно
var count = CalculateFragmentsCount(source, options);
// Проверка, укладывается ли количество тензоров поровну в батчи
int offset = count % options.MaxBatchSize;
// Количество батчей
int count_tensor_batches = count / options.MaxBatchSize + (offset == 0 ? 0 : 1);
// Батчи
var tensors = new ImagePredictionInput[count_tensor_batches];
// Инициализация батчей
Parallel.For(0, count_tensor_batches, batch_index =>
{
if (tensor_part_index > 0 && tensor_part_index % options.MaxBatchSize == 0)
if (batch_index < count_tensor_batches - 1)
{
tensor_index++;
tensor_part_index = 0;
tensors[batch_index] = new ImagePredictionInput
{
Tensor = InitInputTensor(options, options.MaxBatchSize),
Offsets = new OffsetBox[options.MaxBatchSize],
Count = options.MaxBatchSize
};
}
var starty = y;
var dy = (y + options.Crop.Height) - image.Height;
if (dy > 0)
else
{
starty -= dy;
tensors[batch_index] = new ImagePredictionInput
{
Tensor = InitInputTensor(options, offset == 0 ? options.MaxBatchSize : offset),
Offsets = new OffsetBox[offset == 0 ? options.MaxBatchSize : offset],
Count = offset == 0 ? options.MaxBatchSize : offset
};
}
using (var copy = image
.Clone(img => img
.Crop(new Rectangle(startx, starty, options.Crop.Width, options.Crop.Height))
.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
});
// Заполнение батчей
int tensor_index = 0;
// Если используется ресайз оригинала кроме кропов, пишется в первый батч в первый тензор
if (options.Crop.SaveOriginal)
{
using (var copy = source.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
{
FillTensor(tensors[tensor_index].Tensor, copy, tensor_part_index, options, pixToTensor);
tensors[tensor_index].Offsets[tensor_part_index] = new OffsetBox(startx, starty, options.Crop.Width, options.Crop.Height);
FillTensor(tensors[0].Tensor, copy, 0, options, pixToTensor);
tensors[tensor_index].Offsets[0] = new OffsetBox(0, 0, image.Width, image.Height);
}
tensor_part_index++;
tensor_index++;
}
tensor_index--;
Parallel.ForEach(SteppedIterator(0, source.Width, Dx), x =>
{
// Можно запараллелить и тут, но выигрыш дает малоощутимый
for (int y = 0; y < source.Height; y += Dy)
{
var current_index = Interlocked.Increment(ref tensor_index);
// Индекс тензора внутри батча
var b_index = current_index % options.MaxBatchSize;
// Индекс батча
var p_index = (int)Math.Round((double)current_index / (double)options.MaxBatchSize, MidpointRounding.ToNegativeInfinity);
int w = CRw;
if ((x + CRw) > source.Width)
{
w = source.Width - x;
}
int h = CRh;
if ((y + CRh) > source.Height)
{
h = source.Height - y;
}
// Заполнение b_index тензора в p_index батче
FillTensor(tensors[p_index].Tensor, source, x, y, w, h, b_index, options, pixToTensor);
// Указание смещений для данного тензора
tensors[p_index].Offsets[b_index] = new OffsetBox(x, y, options.Crop.Width, options.Crop.Height);
}
});
return tensors;
}
return tensors;
}
// if resize only
result = new PredictionInput[1];
result = new ImagePredictionInput[1];
using (var copy = image.Clone(img => img.Resize(options.InputWidth, options.InputHeight, KnownResamplers.Bicubic)))
{
Tensor<float> tensor = InitInputTensor(options);
FillTensor(tensor, copy, 0, options, pixToTensor);
result[0] = new PredictionInput { Count = 1, Offsets = null, Tensor = tensor };
result[0] = new ImagePredictionInput
{
Count = 1,
Offsets = null,
Tensor = tensor
};
}
return result;
}
private static IEnumerable<int> SteppedIterator(int startIndex, int endIndex, int stepSize)
{
for (int i = startIndex; i < endIndex; i += stepSize)
{
yield return i;
}
}
public static Image Crop(Image source, float x1, float y1, float x2, float y2)
{
int left = 0;

@ -0,0 +1,118 @@
using ZeroLevel.NN.Models;
namespace ZeroLevel.NN.Services
{
public static class NMS
{
private const float IOU_THRESHOLD = .2f;
private const float IOU_MERGE_THRESHOLD = .5f;
public static void Apply(List<YoloPrediction> boxes)
{
for (int i = 0; i < boxes.Count - 1; i++)
{
var left = boxes[i];
for (int j = i + 1; j < boxes.Count; j++)
{
var right = boxes[j];
if (left.Class == right.Class)
{
// удаление вложенных боксов
var ni = NestingIndex(left, right);
if (ni == 1)
{
boxes.RemoveAt(i);
i--;
break;
}
else if (ni == 2)
{
boxes.RemoveAt(j);
j--;
}
// -----------------------------
else
{
var iou = IOU(left, right);
if (iou > IOU_THRESHOLD)
{
if (left.Score > right.Score)
{
boxes.RemoveAt(j);
j--;
}
else
{
boxes.RemoveAt(i);
i--;
break;
}
}
/*else if (threshold > 0.01f && iou > 0.2f)
{
// UNION
var x1 = Math.Min(left.X, right.X);
var y1 = Math.Min(left.Y, right.Y);
var x2 = Math.Max(left.X + left.W, right.X + right.W);
var y2 = Math.Max(left.Y + left.H, right.Y + right.H);
var w = x2 - x1;
var h = y2 - y1;
boxes.RemoveAt(j);
boxes.RemoveAt(i);
boxes.Add(new Prediction
{
Class = left.Class,
Label = left.Label,
Score = Math.Max(left.Score, right.Score),
Cx = x1 + w / 2.0f,
Cy = y1 + h / 2.0f,
W = w,
H = h
});
i--;
break;
}*/
}
}
}
}
}
/// <summary>
/// проверка на вложенность боксов
/// </summary>
static int NestingIndex(YoloPrediction box1, YoloPrediction box2)
{
if (box1.X > box2.X &&
box1.Y > box2.Y &&
(box1.X + box1.W) < (box2.X + box2.W) &&
(box1.Y + box1.H) < (box2.Y + box2.H))
{
return 1;
}
if (box2.X > box1.X &&
box2.Y > box1.Y &&
(box2.X + box2.W) < (box1.X + box1.W) &&
(box2.Y + box2.H) < (box1.Y + box1.H))
{
return 2;
}
return 0;
}
static float IOU(YoloPrediction box1, YoloPrediction box2)
{
var left = (float)Math.Max(box1[0], box2[0]);
var right = (float)Math.Min(box1[2], box2[2]);
var top = (float)Math.Max(box1[1], box2[1]);
var bottom = (float)Math.Min(box1[3], box2[3]);
var width = right - left;
var height = bottom - top;
var intersectionArea = width * height;
return intersectionArea / (float)(box1.Area + box2.Area - intersectionArea);
}
}
}

@ -1,7 +1,6 @@
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using ZeroLevel.NN.Models;
namespace ZeroLevel.NN
@ -18,6 +17,7 @@ namespace ZeroLevel.NN
try
{
var so = SessionOptions.MakeSessionOptionWithCudaProvider(0);
so.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_VERBOSE;
so.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL;
_session = new InferenceSession(modelPath, so);
}
@ -64,7 +64,7 @@ namespace ZeroLevel.NN
vector[i] *= inverseLength;
}
}
protected PredictionInput[] MakeInputBatch(Image<Rgb24> image, ImagePreprocessorOptions options)
protected ImagePredictionInput[] MakeInputBatch(Image image, ImagePreprocessorOptions options)
{
return ImagePreprocessor.ToTensors(image, options);
}
@ -75,6 +75,22 @@ namespace ZeroLevel.NN
return input[0].Tensor;
}
protected int Argmax(float[] embedding)
{
if (embedding.Length == 0) return -1;
var im = 0;
var max = embedding[0];
for (var i = 1; i < embedding.Length; i++)
{
if (embedding[i] > max)
{
im = i;
max = embedding[i];
}
}
return im;
}
public void Dispose()
{
_session?.Dispose();

@ -7,7 +7,7 @@
<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
<DebugType>embedded</DebugType>
<ErrorReport>none</ErrorReport>
<Version>1.0.0.1</Version>
<Version>1.0.0.2</Version>
<Company>Ogoun</Company>
<Authors>Ogoun</Authors>
<Copyright>Copyright Ogoun 2022</Copyright>
@ -15,13 +15,18 @@
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<PackageReleaseNotes>New architectures</PackageReleaseNotes>
<PackageReleaseNotes></PackageReleaseNotes>
<Platforms>AnyCPU;x64</Platforms>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
<Optimize>False</Optimize>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Optimize>False</Optimize>
</PropertyGroup>
<ItemGroup>
<None Include="..\zero.png">
<Pack>True</Pack>
@ -30,10 +35,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Aurigma.GraphicsMill.Core.x64" Version="10.5.308" />
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.10.0" />
<PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.10.0" />
<PackageReference Include="SixLabors.ImageSharp" Version="2.1.0" />
<PackageReference Include="Aurigma.GraphicsMill.Core.x64" Version="10.6.25" />
<PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.11.0" />
<PackageReference Include="SixLabors.ImageSharp" Version="2.1.3" />
<PackageReference Include="SixLabors.ImageSharp.Drawing" Version="1.0.0-beta14" />
</ItemGroup>
<ItemGroup>

@ -1,77 +1,94 @@
using System;
using System.Data.SQLite;
using SQLite;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq.Expressions;
using ZeroLevel.Services.FileSystem;
namespace ZeroLevel.SqLite
{
public abstract class BaseSqLiteDB
public abstract class BaseSqLiteDB<T>
: IDisposable
where T : class, new()
{
#region Helpers
protected static bool HasColumn(SQLiteDataReader dr, string columnName)
protected SQLiteConnection _db;
public BaseSqLiteDB(string name)
{
for (int i = 0; i < dr.FieldCount; i++)
{
if (dr.GetName(i).Equals(columnName, StringComparison.InvariantCultureIgnoreCase))
return true;
}
return false;
_db = new SQLiteConnection(PrepareDb(name));
}
protected static Tr Read<Tr>(SQLiteDataReader reader, int index)
public T Append(T record)
{
if (reader == null || reader.FieldCount <= index || reader[index] == DBNull.Value) return default;
Type t;
if ((t = Nullable.GetUnderlyingType(typeof(Tr))) != null)
{
return (Tr)Convert.ChangeType(reader[index], t);
}
return (Tr)Convert.ChangeType(reader[index], typeof(Tr));
_db.Insert(record);
return record;
}
protected static Tr Read<Tr>(SQLiteDataReader reader, string name)
public void CreateTable()
{
if (reader == null || !HasColumn(reader, name) || reader[name] == DBNull.Value) return default;
Type t;
if ((t = Nullable.GetUnderlyingType(typeof(Tr))) != null)
{
return (Tr)Convert.ChangeType(reader[name], t);
}
return (Tr)Convert.ChangeType(reader[name], typeof(Tr));
_db.CreateTable<T>();
}
protected static void Execute(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null)
public void DropTable()
{
using (var cmd = new SQLiteCommand(query, connection))
{
if (parameters != null && parameters.Length > 0)
{
cmd.Parameters.AddRange(parameters);
}
cmd.ExecuteNonQuery();
}
_db.DropTable<T>();
}
protected static object ExecuteScalar(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null)
public IEnumerable<T> SelectAll()
{
using (var cmd = new SQLiteCommand(query, connection))
return _db.Table<T>();
}
public IEnumerable<T> SelectBy(Expression<Func<T, bool>> predicate)
{
return _db.Table<T>().Where(predicate);
}
public T Single(Expression<Func<T, bool>> predicate)
{
return _db.Table<T>().FirstOrDefault(predicate);
}
public T Single<U>(Expression<Func<T, bool>> predicate, Expression<Func<T, U>> orderBy, bool desc = false)
{
if (desc)
{
if (parameters != null && parameters.Length > 0)
{
cmd.Parameters.AddRange(parameters);
}
return cmd.ExecuteScalar();
return _db.Table<T>().Where(predicate).OrderByDescending(orderBy).FirstOrDefault();
}
return _db.Table<T>().Where(predicate).OrderBy(orderBy).FirstOrDefault();
}
protected static SQLiteDataReader Read(string query, SQLiteConnection connection, SQLiteParameter[] parameters = null)
public T Single<U>(Expression<Func<T, U>> orderBy, bool desc = false)
{
using (var cmd = new SQLiteCommand(query, connection))
if (desc)
{
if (parameters != null && parameters.Length > 0)
{
cmd.Parameters.AddRange(parameters);
}
return cmd.ExecuteReader();
return _db.Table<T>().OrderByDescending(orderBy).FirstOrDefault();
}
return _db.Table<T>().OrderBy(orderBy).FirstOrDefault();
}
public IEnumerable<T> SelectBy(int N, Expression<Func<T, bool>> predicate)
{
return _db.Table<T>().Where(predicate).Take(N);
}
public long Count()
{
return _db.Table<T>().Count();
}
public long Count(Expression<Func<T, bool>> predicate)
{
return _db.Table<T>().Count(predicate);
}
public int Delete(Expression<Func<T, bool>> predicate)
{
return _db.Table<T>().Delete(predicate);
}
public void Update(T record)
{
_db.Update(record);
}
protected static string PrepareDb(string path)
@ -80,13 +97,23 @@ namespace ZeroLevel.SqLite
{
path = Path.Combine(FSUtils.GetAppLocalDbDirectory(), path);
}
if (!File.Exists(path))
{
SQLiteConnection.CreateFile(path);
}
return Path.GetFullPath(path);
}
#endregion Helpers
protected abstract void DisposeStorageData();
public void Dispose()
{
DisposeStorageData();
try
{
_db?.Close();
_db?.Dispose();
}
catch (Exception ex)
{
Log.Error(ex, "[BaseSqLiteDB] Fault close db connection");
}
}
}
}

@ -1,10 +1,15 @@
namespace ZeroLevel.SqLite
using SQLite;
namespace ZeroLevel.SqLite
{
public class User
{
[PrimaryKey, AutoIncrement]
public long Id { get; set; }
[Indexed]
public string UserName { get; set; }
public string DisplayName { get; set; }
[Indexed]
public byte[] PasswordHash { get; set; }
public long Timestamp { get; set; }
public long Creator { get; set; }

@ -1,24 +1,28 @@
using System;
using System.Collections.Generic;
using System.Data.SQLite;
using System.IO;
using System.Text;
using SQLite;
using System;
using System.Threading;
using ZeroLevel.Services.Serialization;
using ZeroLevel.Services.Shedulling;
namespace ZeroLevel.SqLite
{
public sealed class ExpirationRecord
{
[PrimaryKey, AutoIncrement]
public long Id { get; set; }
[Indexed]
public long Expiration { get; set; }
public byte[] Data { get; set; }
}
public sealed class SqLiteDelayDataStorage<T>
: BaseSqLiteDB, IDisposable
where T : IBinarySerializable
: BaseSqLiteDB<ExpirationRecord>
where T : class, IBinarySerializable, new()
{
#region Fields
private readonly IExpirationSheduller _sheduller;
private readonly Func<T, DateTime> _expire_date_calc_func;
private readonly SQLiteConnection _db;
private readonly string _table_name;
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
#endregion Fields
@ -28,14 +32,10 @@ namespace ZeroLevel.SqLite
public SqLiteDelayDataStorage(string database_file_path,
Func<T, bool> expire_callback,
Func<T, DateTime> expire_date_calc_func)
: base(database_file_path)
{
this._expire_date_calc_func = expire_date_calc_func;
var path = PrepareDb(database_file_path);
_table_name = "expiration";
_db = new SQLiteConnection($"Data Source={path};Version=3;");
_db.Open();
Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB, expirationtime INTEGER)", _db);
Execute($"CREATE INDEX IF NOT EXISTS expirationtime_index ON {_table_name} (expirationtime)", _db);
CreateTable();
_sheduller = Sheduller.CreateExpirationSheduller();
OnExpire += expire_callback;
Preload();
@ -64,13 +64,8 @@ namespace ZeroLevel.SqLite
long id = -1;
try
{
Execute($"INSERT INTO {_table_name} ('body', 'expirationtime') values (@body, @expirationtime)", _db,
new SQLiteParameter[]
{
new SQLiteParameter("body", MessageSerializer.Serialize(packet)),
new SQLiteParameter("expirationtime", expirationTime)
});
id = (long)ExecuteScalar("select last_insert_rowid();", _db);
var r = Append(new ExpirationRecord { Expiration = expirationTime, Data = MessageSerializer.Serialize(packet) });
id = r.Id;
}
catch (Exception ex)
{
@ -91,17 +86,13 @@ namespace ZeroLevel.SqLite
private void Preload()
{
SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
reader = Read($"SELECT id, expirationtime FROM {_table_name}", _db);
while (reader.Read())
foreach (var record in SelectAll())
{
var id = reader.GetInt64(0);
_sheduller.Push(new DateTime(reader.GetInt64(1), DateTimeKind.Local), (k) => Pop(id));
_sheduller.Push(new DateTime(record.Expiration, DateTimeKind.Local), (k) => Pop(record.Id));
}
reader.Close();
}
catch (Exception ex)
{
@ -111,7 +102,6 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
reader = null;
}
private void Pop(long id)
@ -122,7 +112,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterReadLock();
try
{
body = (byte[])ExecuteScalar($"SELECT body FROM {_table_name} WHERE id=@id", _db, new SQLiteParameter[] { new SQLiteParameter("id", id) });
body = Single(r=>r.Id == id)?.Data;
}
catch (Exception ex)
{
@ -161,8 +151,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
Execute($"DELETE FROM {_table_name} WHERE id = @id", _db,
new SQLiteParameter[] { new SQLiteParameter("id", id) });
Delete(r => r.Id == id);
}
catch (Exception ex)
{
@ -178,17 +167,8 @@ namespace ZeroLevel.SqLite
#region IDisposable
public void Dispose()
protected override void DisposeStorageData()
{
try
{
_db?.Close();
_db?.Dispose();
}
catch (Exception ex)
{
Log.Error(ex, "[SqLiteDelayDataStorage] Fault close db connection");
}
_sheduller.Dispose();
}

@ -1,6 +1,6 @@
using System;
using SQLite;
using System;
using System.Collections.Generic;
using System.Data.SQLite;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
@ -8,40 +8,37 @@ using System.Threading;
namespace ZeroLevel.SqLite
{
public sealed class DuplicateRecord
{
[PrimaryKey, AutoIncrement]
public long Id { get; set; }
[Indexed]
public string Hash { get; set; }
public long Timestamp { get; set; }
public byte[] Data { get; set; }
}
/// <summary>
/// Хранит данные указанное число дней, и позволяет выполнить проверку наличия данных, для отбрасывания дубликатов
/// </summary>
public sealed class SqLiteDupStorage
: BaseSqLiteDB, IDisposable
: BaseSqLiteDB<DuplicateRecord>
{
#region Fields
private const string DEFAUL_TABLE_NAME = "History";
private readonly SQLiteConnection _db;
private readonly long _removeOldRecordsTaskKey;
private readonly int _countDays;
private readonly string _table_name;
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
#endregion Fields
#region Private members
private sealed class DuplicationStorageRecord
{
public string Hash { get; set; }
public byte[] Body { get; set; }
public long Timestamp { get; set; }
}
private void RemoveOldRecordsTask(long key)
{
_rwLock.EnterWriteLock();
try
{
Execute($"DELETE FROM {_table_name} WHERE timestamp < @limit", _db,
new SQLiteParameter[] { new SQLiteParameter("limit", DateTime.Now.AddDays(-_countDays).Ticks) });
Delete(r => r.Timestamp < DateTime.Now.AddDays(-_countDays).Ticks);
}
catch (Exception ex)
{
@ -57,21 +54,10 @@ namespace ZeroLevel.SqLite
#region Ctor
public SqLiteDupStorage(string database_file_path, string tableName, int period)
public SqLiteDupStorage(string database_file_path, int period)
: base(database_file_path)
{
var path = PrepareDb(database_file_path);
if (string.IsNullOrWhiteSpace(tableName))
{
_table_name = DEFAUL_TABLE_NAME;
}
else
{
_table_name = tableName;
}
_db = new SQLiteConnection($"Data Source={path};Version=3;");
_db.Open();
Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, body BLOB, timestamp INTEGER)", _db);
Execute($"CREATE INDEX IF NOT EXISTS hash_index ON {_table_name} (hash)", _db);
CreateTable();
_countDays = period > 0 ? period : 1;
_removeOldRecordsTaskKey = Sheduller.RemindEvery(TimeSpan.FromMinutes(1), RemoveOldRecordsTask);
}
@ -87,17 +73,14 @@ namespace ZeroLevel.SqLite
{
var hash = GenerateSHA256String(body);
var timestamp = DateTime.Now.Ticks;
SQLiteDataReader reader;
_rwLock.EnterReadLock();
var exists = new List<byte[]>();
try
{
reader = Read($"SELECT body FROM {_table_name} WHERE hash=@hash", _db, new SQLiteParameter[] { new SQLiteParameter("hash", hash) });
while (reader.Read())
foreach (var record in SelectBy(r => r.Hash == hash))
{
exists.Add((byte[])reader.GetValue(0));
exists.Add(record.Data);
}
reader.Close();
}
catch (Exception ex)
{
@ -108,7 +91,6 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
reader = null;
if (exists.Any())
{
foreach (var candidate in exists)
@ -120,13 +102,12 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
Execute($"INSERT INTO {_table_name} ('hash', 'body', 'timestamp') values (@hash, @body, @timestamp)", _db,
new SQLiteParameter[]
{
new SQLiteParameter("hash", hash),
new SQLiteParameter("body", body),
new SQLiteParameter("timestamp", timestamp)
});
Append(new DuplicateRecord
{
Data = body,
Hash = hash,
Timestamp = timestamp
});
}
catch (Exception ex)
{
@ -143,17 +124,9 @@ namespace ZeroLevel.SqLite
#region IDisposable
public void Dispose()
protected override void DisposeStorageData()
{
Sheduller.Remove(_removeOldRecordsTaskKey);
try
{
_db?.Dispose();
}
catch (Exception ex)
{
Log.Error(ex, "[SQLiteDupStorage] Fault close db connection");
}
}
#endregion IDisposable

@ -1,15 +1,25 @@
using System;
using System.Data.SQLite;
using SQLite;
using System;
using System.Threading;
using ZeroLevel.Services.Serialization;
namespace ZeroLevel.SqLite
{
public sealed class PacketRecord
{
[PrimaryKey, AutoIncrement]
public long Id { get; set; }
[Indexed]
public long Timestamp { get; set; }
public byte[] Data { get; set; }
}
/// <summary>
/// Промежуточное/временное хранилище пакетов данных, для случаев сбоя доставок через шину данных
/// </summary>
public sealed class SqLitePacketBuffer<T>
: BaseSqLiteDB, IDisposable
: BaseSqLiteDB<PacketRecord>
where T : IBinarySerializable
{
private sealed class PacketBufferRecord
@ -20,20 +30,14 @@ namespace ZeroLevel.SqLite
#region Fields
private readonly SQLiteConnection _db;
private readonly string _table_name;
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
#endregion Fields
public SqLitePacketBuffer(string database_file_path)
: base(database_file_path)
{
var path = PrepareDb(database_file_path);
_table_name = "packets";
_db = new SQLiteConnection($"Data Source={path};Version=3;");
_db.Open();
Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB, created INTEGER)", _db);
Execute($"CREATE INDEX IF NOT EXISTS created_index ON {_table_name} (created)", _db);
CreateTable();
}
public void Push(T frame)
@ -43,13 +47,11 @@ namespace ZeroLevel.SqLite
var creationTime = DateTime.Now.Ticks;
try
{
Execute($"INSERT INTO {_table_name} ('body', 'created') values (@body, @created)", _db,
new SQLiteParameter[]
{
new SQLiteParameter("body", MessageSerializer.Serialize(frame)),
new SQLiteParameter("created", creationTime)
});
id = (long)ExecuteScalar("select last_insert_rowid();", _db);
id = Append(new PacketRecord
{
Data = MessageSerializer.Serialize(frame),
Timestamp = creationTime
}).Id;
}
catch (Exception ex)
{
@ -65,25 +67,20 @@ namespace ZeroLevel.SqLite
{
bool success = false;
long id = -1;
SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
reader = Read($"SELECT id, body FROM {_table_name} ORDER BY created ASC LIMIT 1", _db);
if (reader.Read())
var record = Single(r => r.Timestamp);
id = record.Id;
var body = record.Data;
try
{
success = pop_callback(MessageSerializer.Deserialize<T>(body));
}
catch (Exception ex)
{
id = reader.GetInt64(0);
var body = (byte[])reader.GetValue(1);
try
{
success = pop_callback(MessageSerializer.Deserialize<T>(body));
}
catch (Exception ex)
{
Log.Error(ex, "Fault handle buffered data");
}
Log.Error(ex, "Fault handle buffered data");
}
reader.Close();
}
catch (Exception ex)
{
@ -97,7 +94,6 @@ namespace ZeroLevel.SqLite
{
RemoveRecordById(id);
}
reader = null;
return success;
}
@ -106,8 +102,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
Execute($"DELETE FROM {_table_name} WHERE id = @id", _db,
new SQLiteParameter[] { new SQLiteParameter("id", id) });
Delete(r => r.Id == id);
}
catch (Exception ex)
{
@ -119,17 +114,8 @@ namespace ZeroLevel.SqLite
}
}
public void Dispose()
protected override void DisposeStorageData()
{
try
{
_db?.Close();
_db?.Dispose();
}
catch (Exception ex)
{
Log.Error(ex, "[SqLitePacketBuffer] Fault close db connection");
}
}
}
}

@ -1,18 +1,15 @@
using System;
using System.Collections.Generic;
using System.Data.SQLite;
using System.Threading;
using ZeroLevel.Models;
namespace ZeroLevel.SqLite
{
public class SqLiteUserRepository
: BaseSqLiteDB
: BaseSqLiteDB<User>
{
#region Fields
private readonly SQLiteConnection _db;
private readonly string _table_name = "users";
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
#endregion Fields
@ -20,13 +17,8 @@ namespace ZeroLevel.SqLite
#region Ctor
public SqLiteUserRepository()
: base("users.db")
{
var path =PrepareDb("users.db");
_db = new SQLiteConnection($"Data Source={path};Version=3;");
_db.Open();
Execute($"CREATE TABLE IF NOT EXISTS {_table_name} (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT, displayname TEXT, hash BLOB, timestamp INTEGER, creator INTEGER, role INTEGER)", _db);
Execute($"CREATE INDEX IF NOT EXISTS username_index ON {_table_name} (username)", _db);
Execute($"CREATE INDEX IF NOT EXISTS hash_index ON {_table_name} (hash)", _db);
}
#endregion Ctor
@ -34,25 +26,13 @@ namespace ZeroLevel.SqLite
public IEnumerable<User> GetAll()
{
var list = new List<User>();
SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name}", _db);
while (reader.Read())
foreach (var r in SelectAll())
{
list.Add(new User
{
Id = reader.GetInt64(0),
UserName = reader.GetString(1),
DisplayName = Read<string>(reader, 2),
PasswordHash = (byte[])reader.GetValue(3),
Timestamp = reader.GetInt64(4),
Creator = reader.GetInt64(5),
Role = (UserRole)reader.GetInt32(6)
});
list.Add(r);
}
reader.Close();
}
catch (Exception ex)
{
@ -62,34 +42,16 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
reader = null;
return list;
}
public User Get(long id)
{
User user = null;
SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name} WHERE id = @id", _db,
new SQLiteParameter[] { new SQLiteParameter("id", id) });
if (reader.Read())
{
var body = (byte[])reader.GetValue(1);
user = new User
{
Id = reader.GetInt64(0),
UserName = reader.GetString(1),
DisplayName = reader.GetString(2),
PasswordHash = (byte[])reader.GetValue(3),
Timestamp = reader.GetInt64(4),
Creator = reader.GetInt64(5),
Role = (UserRole)reader.GetInt32(6)
};
}
reader.Close();
user = Single(r => r.Id == id);
}
catch (Exception ex)
{
@ -99,37 +61,16 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
reader = null;
return user;
}
public User Get(string username, byte[] hash)
{
User user = null;
SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
reader = Read($"SELECT id, username, displayname, hash, timestamp, creator, role FROM {_table_name} WHERE username = @username AND hash = @hash", _db,
new SQLiteParameter[]
{
new SQLiteParameter("username", username),
new SQLiteParameter("hash", hash)
});
if (reader.Read())
{
user = new User
{
Id = reader.GetInt64(0),
UserName = reader.GetString(1),
DisplayName = reader.GetString(2),
PasswordHash = (byte[])reader.GetValue(3),
Timestamp = reader.GetInt64(4),
Creator = reader.GetInt64(5),
Role = (UserRole)reader.GetInt32(6)
};
}
reader.Close();
user = Single(r => r.UserName == username && r.PasswordHash == hash);
}
catch (Exception ex)
{
@ -139,7 +80,6 @@ namespace ZeroLevel.SqLite
{
_rwLock.ExitReadLock();
}
reader = null;
return user;
}
@ -150,22 +90,12 @@ namespace ZeroLevel.SqLite
var creationTime = DateTime.UtcNow.Ticks;
try
{
var count_obj = ExecuteScalar($"SELECT COUNT(*) FROM {_table_name} WHERE username=@username", _db, new SQLiteParameter[] { new SQLiteParameter("username", user.UserName) });
if (count_obj != null && (long)count_obj > 0)
var count_obj = Count(r => r.UserName == user.UserName);
if (count_obj > 0)
{
return InvokeResult<long>.Fault<long>("Пользователь уже существует");
return InvokeResult<long>.Fault<long>("Пользователь с таким именем уже существует");
}
Execute($"INSERT INTO {_table_name} ('username', 'displayname', 'hash', 'timestamp', 'creator', 'role') values (@username, @displayname, @hash, @timestamp, @creator, @role)", _db,
new SQLiteParameter[]
{
new SQLiteParameter("username", user.UserName),
new SQLiteParameter("displayname", user.DisplayName),
new SQLiteParameter("hash", user.PasswordHash),
new SQLiteParameter("timestamp", creationTime),
new SQLiteParameter("creator", user.Creator),
new SQLiteParameter("role", user.Role)
});
id = (long)ExecuteScalar("select last_insert_rowid();", _db);
id = Append(user).Id;
}
catch (Exception ex)
{
@ -184,8 +114,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
Execute($"DELETE FROM {_table_name} WHERE username = @username", _db,
new SQLiteParameter[] { new SQLiteParameter("username", login) });
Delete(r => r.UserName == login);
return InvokeResult.Succeeding();
}
catch (Exception ex)
@ -198,5 +127,9 @@ namespace ZeroLevel.SqLite
_rwLock.ExitWriteLock();
}
}
protected override void DisposeStorageData()
{
}
}
}

@ -1,36 +1,36 @@
using System;
using SQLite;
using System;
using System.Collections.Generic;
using System.Data.SQLite;
using System.Runtime.CompilerServices;
using System.Threading;
using ZeroLevel.Services.Serialization;
namespace ZeroLevel.SqLite
{
public sealed class DataRecord
{
[PrimaryKey, AutoIncrement]
public long Id { get; set; }
[Indexed]
public string Key { get; set; }
public byte[] Data { get; set; }
}
public sealed class UserCacheRepository<T>
: BaseSqLiteDB, IDisposable
: BaseSqLiteDB<DataRecord>
where T : IBinarySerializable
{
#region Fields
private readonly SQLiteConnection _db;
private readonly ReaderWriterLockSlim _rwLock = new ReaderWriterLockSlim();
private readonly string _tableName;
#endregion Fields
#region Ctor
public UserCacheRepository()
: base(typeof(T).Name)
{
_tableName = typeof(T).Name;
var path = PrepareDb($"{_tableName}_user_cachee.db");
_db = new SQLiteConnection($"Data Source={path};Version=3;");
_db.Open();
Execute($"CREATE TABLE IF NOT EXISTS {_tableName} (id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT, body BLOB)", _db);
Execute($"CREATE INDEX IF NOT EXISTS key_index ON {_tableName} (key)", _db);
CreateTable();
}
#endregion Ctor
@ -47,8 +47,8 @@ namespace ZeroLevel.SqLite
_rwLock.EnterReadLock();
try
{
var count_obj = ExecuteScalar($"SELECT COUNT(*) FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[] { new SQLiteParameter("key", key) });
if (count_obj != null && (long)count_obj > 0)
var count_obj = Count(r => r.Key == key);
if (count_obj > 0)
{
update = true;
}
@ -68,20 +68,16 @@ namespace ZeroLevel.SqLite
var body = MessageSerializer.Serialize(data);
if (update)
{
Execute($"UPDATE {_tableName} SET body=@body WHERE key=@key", _db,
new SQLiteParameter[]
{
new SQLiteParameter("key", key),
new SQLiteParameter("body", body)
});
var r = Single(r => r.Key == key);
r.Data = body;
Update(r);
}
else
{
Execute($"INSERT INTO {_tableName} ('key', 'body') values (@key, @body)", _db,
new SQLiteParameter[]
Append(new DataRecord
{
new SQLiteParameter("key", key),
new SQLiteParameter("body", body)
Data = body,
Key = key
});
}
return true;
@ -103,10 +99,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
Execute($"DELETE FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[]
{
new SQLiteParameter("key", key)
});
Delete(r => r.Key == key);
}
catch (Exception ex)
{
@ -124,10 +117,7 @@ namespace ZeroLevel.SqLite
_rwLock.EnterWriteLock();
try
{
return Convert.ToInt64(ExecuteScalar($"SELECT COUNT(*) FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[]
{
new SQLiteParameter("key", key)
}));
return Count(r => r.Key == key);
}
catch (Exception ex)
{
@ -144,23 +134,17 @@ namespace ZeroLevel.SqLite
{
var key = KEY(userid, name);
var result = new List<T>();
SQLiteDataReader reader;
_rwLock.EnterReadLock();
try
{
reader = Read($"SELECT [body] FROM {_tableName} WHERE key=@key", _db, new SQLiteParameter[]
foreach (var r in SelectBy(r=>r.Key == key))
{
new SQLiteParameter("key", key)
});
while (reader.Read())
{
var data = Read<byte[]>(reader, 0);
var data = r.Data;
if (data != null)
{
result.Add(MessageSerializer.Deserialize<T>(data));
}
}
reader.Close();
}
catch (Exception ex)
{
@ -169,7 +153,6 @@ namespace ZeroLevel.SqLite
finally
{
_rwLock.ExitReadLock();
reader = null;
}
return result;
}
@ -177,17 +160,9 @@ namespace ZeroLevel.SqLite
#endregion API
#region IDisposable
public void Dispose()
protected override void DisposeStorageData()
{
try
{
_db?.Dispose();
}
catch (Exception ex)
{
Log.Error(ex, "[UserCacheRepository] Fault close db connection");
}
}
#endregion IDisposable

@ -13,9 +13,9 @@ Based on System.Data.SQLite.Core</Description>
<PackageIconUrl />
<Platforms>AnyCPU;x64;x86</Platforms>
<PackageReleaseNotes>Move to new ZeroLevel version</PackageReleaseNotes>
<AssemblyVersion>1.0.3.0</AssemblyVersion>
<FileVersion>1.0.3.0</FileVersion>
<Version>1.0.3.0</Version>
<AssemblyVersion>1.0.4.0</AssemblyVersion>
<FileVersion>1.0.4.0</FileVersion>
<Version>1.0.4.0</Version>
<OutputType>Library</OutputType>
</PropertyGroup>
@ -27,11 +27,17 @@ Based on System.Data.SQLite.Core</Description>
</ItemGroup>
<ItemGroup>
<PackageReference Include="System.Data.SQLite.Core" Version="1.0.115" />
<PackageReference Include="sqlite-net-static" Version="1.8.116" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ZeroLevel\ZeroLevel.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="sqlite3.dll">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

Binary file not shown.

@ -0,0 +1,42 @@
using Xunit;
using ZeroLevel.Semantic;
using ZeroLevel.Services.Text;
namespace ZeroLevel.UnitTests
{
public class SuffixAutomataTests
{
[Fact]
public void IsSubstringTest()
{
var text = @"=Однако эту оценку легко показать и без знания алгоритма. Вспомним о том, что число состояний равно количеству различных значений множеств endpos.#";
var automata = new SuffixAutomata();
automata.Init();
foreach (var ch in text)
{
automata.Extend(ch);
}
Assert.True(automata.IsSubstring("Вспомним"));
Assert.True(automata.IsSubstring("")); // empty line
Assert.True(automata.IsSubstring("#")); // end line
Assert.True(automata.IsSubstring("=")); // start line
Assert.True(automata.IsSubstring(null)); // null
Assert.False(automata.IsSubstring("равноценно"));
Assert.False(automata.IsSubstring("нетслова"));
}
[Fact]
public void IntersectionTest()
{
var text = @"=Однако эту оценку легко показать и без знания алгоритма. Вспомним о том, что число состояний равно количеству различных значений множеств endpos.#";
var i = LongestCommonSubstring.LCS(text, "енк");
Assert.Equal("енк", i);
i = LongestCommonSubstring.LCS(text, "стоя");
Assert.Equal("стоя", i);
i = LongestCommonSubstring.LCS(text, "горизонт");
Assert.Equal("гори", i);
}
}
}

@ -39,8 +39,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "WPFExamples", "WPFExamples"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ConnectionTest", "ConnectionTest", "{D5207A5A-2F27-4992-9BA5-0BDCFC59F133}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Client", "ConnectionTest\Client\Client.csproj", "{08CDD42E-E324-40A4-88C3-EDD0493AAF84}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Server", "ConnectionTest\Server\Server.csproj", "{3496A688-0749-48C2-BD60-ABB42A5C17C9}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ZeroLevel.Qdrant", "ZeroLevel.Qdrant\ZeroLevel.Qdrant.csproj", "{7188B89E-96EB-4EFB-AAFB-D0A823031F99}"
@ -63,7 +61,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Source", "TestPipeLine\Sour
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Watcher", "TestPipeLine\Watcher\Watcher.csproj", "{F70842E7-9A1D-4CC4-9F55-0953AEC9C7C8}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ZeroLevel.NN", "ZeroLevel.NN\ZeroLevel.NN.csproj", "{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ZeroLevel.NN", "ZeroLevel.NN\ZeroLevel.NN.csproj", "{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Client", "ConnectionTest\Client\Client.csproj", "{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AutoLoader", "AutoLoader", "{2EF83101-63BC-4397-A005-A747189143D4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AppContainer", "AutoLoader\AppContainer\AppContainer.csproj", "{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -171,18 +175,6 @@ Global
{04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x64.Build.0 = Release|x64
{04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x86.ActiveCfg = Release|x86
{04219F58-4D3A-4707-82A8-4DDDC9882969}.Release|x86.Build.0 = Release|x86
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|Any CPU.Build.0 = Debug|Any CPU
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x64.ActiveCfg = Debug|x64
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x64.Build.0 = Debug|x64
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x86.ActiveCfg = Debug|x86
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Debug|x86.Build.0 = Debug|x86
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|Any CPU.ActiveCfg = Release|Any CPU
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|Any CPU.Build.0 = Release|Any CPU
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x64.ActiveCfg = Release|x64
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x64.Build.0 = Release|x64
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x86.ActiveCfg = Release|x86
{08CDD42E-E324-40A4-88C3-EDD0493AAF84}.Release|x86.Build.0 = Release|x86
{3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3496A688-0749-48C2-BD60-ABB42A5C17C9}.Debug|x64.ActiveCfg = Debug|x64
@ -327,12 +319,35 @@ Global
{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x64.Build.0 = Release|Any CPU
{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x86.ActiveCfg = Release|Any CPU
{C67E5F2E-B62E-441D-99F5-8ECA6CECE804}.Release|x86.Build.0 = Release|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x64.ActiveCfg = Debug|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x64.Build.0 = Debug|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x86.ActiveCfg = Debug|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Debug|x86.Build.0 = Debug|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|Any CPU.Build.0 = Release|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x64.ActiveCfg = Release|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x64.Build.0 = Release|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x86.ActiveCfg = Release|Any CPU
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25}.Release|x86.Build.0 = Release|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x64.ActiveCfg = Debug|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x64.Build.0 = Debug|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x86.ActiveCfg = Debug|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Debug|x86.Build.0 = Debug|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|Any CPU.Build.0 = Release|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x64.ActiveCfg = Release|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x64.Build.0 = Release|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x86.ActiveCfg = Release|Any CPU
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{08CDD42E-E324-40A4-88C3-EDD0493AAF84} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133}
{3496A688-0749-48C2-BD60-ABB42A5C17C9} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133}
{54BB2BA9-DAC0-4162-8BC0-E4A9B898CBB0} = {FC074553-5D9F-4DF1-9130-7092E37DE768}
{3D0FE0BA-F7B1-4A63-BBA4-C96514A68426} = {FC074553-5D9F-4DF1-9130-7092E37DE768}
@ -340,6 +355,8 @@ Global
{5CCFF557-C91F-4DD7-9530-D76FE517DA98} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15}
{82202433-6426-4737-BAB2-473AC1F74C5D} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15}
{F70842E7-9A1D-4CC4-9F55-0953AEC9C7C8} = {03ACF314-93FC-46FE-9FB8-3F46A01A5A15}
{2C33D5A3-6CD4-4AAA-A716-B3CD65036E25} = {D5207A5A-2F27-4992-9BA5-0BDCFC59F133}
{9DE345EA-955B-41A8-93AF-277C0B5A9AC5} = {2EF83101-63BC-4397-A005-A747189143D4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A65DB16F-877D-4586-A9F3-8BBBFBAF5CEB}

@ -4,6 +4,21 @@ namespace System.Linq
{
public static class LinqExtension
{
public static void ThrowIfNull<T>(this T val, string message = null)
{
if (null == val)
throw new ArgumentNullException(message);
}
public static void Apply<T>(this IEnumerable<T> seq, Action<T> action)
{
seq.ThrowIfNull();
action.ThrowIfNull();
foreach (var e in seq)
{
action.Invoke(e);
}
}
public static IEnumerable<T[]> ZipLongest<T>(this IEnumerable<T> left, IEnumerable<T> right)
{
IEnumerator<T> leftEnumerator = left.GetEnumerator();

@ -341,6 +341,27 @@ namespace ZeroLevel.Services.FileSystem
var zipFile = Path.Combine(tmp.FullName, "zip.zip");
File.WriteAllBytes(zipFile, data);
ZipFile.ExtractToDirectory(zipFile, targetFolder);
File.Delete(zipFile);
}
public static void UnPackFolder(string zipFile, string targetFolder)
{
if (Directory.Exists(targetFolder))
{
try
{
FSUtils.RemoveFolder(targetFolder, 3, 3000);
}
catch (Exception ex)
{
Log.SystemError(ex, $"[FSUtils] Fault clean folder '{Path.GetDirectoryName(targetFolder)}'");
}
}
if (Directory.Exists(targetFolder) == false)
{
Directory.CreateDirectory(targetFolder);
}
ZipFile.ExtractToDirectory(zipFile, targetFolder);
}
public static void CopyDir(string sourceFolder, string targetFolder)

@ -6,22 +6,23 @@
/// </summary>
public static class StringHash
{
public static uint DotNetFullHash(string str)
public static int DotNetFullHash(string str)
{
unchecked
{
int hash1 = (5381 << 16) + 5381;
int hash2 = hash1;
for (int i = 0; i < str.Length; i += 2)
if (str != null)
{
hash1 = ((hash1 << 5) + hash1) ^ str[i];
if (i == str.Length - 1)
break;
hash2 = ((hash2 << 5) + hash2) ^ str[i + 1];
for (int i = 0; i < str.Length; i += 2)
{
hash1 = ((hash1 << 5) + hash1) ^ str[i];
if (i == str.Length - 1)
break;
hash2 = ((hash2 << 5) + hash2) ^ str[i + 1];
}
}
return (uint)(hash1 + (hash2 * 1566083941)) & 0x7FFFFFFF;
return (hash1 + (hash2 * 1566083941)) & 0x7FFFFFFF;
}
}

@ -15,4 +15,15 @@ namespace MemoryPools
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Return(T instance) => _freeObjectsQueue.Push(instance);
}
public class JetValPool<T>
{
private readonly JetStack<T> _freeObjectsQueue = new JetStack<T>();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public T Get() => _freeObjectsQueue.Count > 0 ? _freeObjectsQueue.Pop() : default(T);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Return(T instance) => _freeObjectsQueue.Push(instance);
}
}

@ -13,8 +13,8 @@ namespace ZeroLevel.Network
IRouter Router { get; }
bool Send(Frame data);
bool Request(Frame data, Action<byte[]> callback, Action<string> fail = null);
bool Response(byte[] data, int identity);
void Send(Frame data);
void Request(Frame data, Action<byte[]> callback, Action<string> fail = null);
void Response(byte[] data, int identity);
}
}

@ -22,7 +22,8 @@ namespace ZeroLevel.Network
{
try
{
return _client.Send(FrameFactory.Create(inbox));
_client.Send(FrameFactory.Create(inbox));
return true;
}
catch (Exception ex)
{
@ -35,7 +36,8 @@ namespace ZeroLevel.Network
{
try
{
return _client.Send(FrameFactory.Create(inbox, data));
_client.Send(FrameFactory.Create(inbox, data));
return true;
}
catch (Exception ex)
{
@ -47,8 +49,9 @@ namespace ZeroLevel.Network
public bool Send<T>(T message)
{
try
{
return _client.Send(FrameFactory.Create(BaseSocket.DEFAULT_MESSAGE_INBOX, MessageSerializer.SerializeCompatible<T>(message)));
{
_client.Send(FrameFactory.Create(BaseSocket.DEFAULT_MESSAGE_INBOX, MessageSerializer.SerializeCompatible<T>(message)));
return true;
}
catch (Exception ex)
{
@ -60,8 +63,9 @@ namespace ZeroLevel.Network
public bool Send<T>(string inbox, T message)
{
try
{
return _client.Send(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible<T>(message)));
{
_client.Send(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible<T>(message)));
return true;
}
catch (Exception ex)
{
@ -73,8 +77,9 @@ namespace ZeroLevel.Network
public bool Request(string inbox, Action<byte[]> callback)
{
try
{
return _client.Request(FrameFactory.Create(inbox), f => callback(f));
{
_client.Request(FrameFactory.Create(inbox), f => callback(f));
return true;
}
catch (Exception ex)
{
@ -87,7 +92,8 @@ namespace ZeroLevel.Network
{
try
{
return _client.Request(FrameFactory.Create(inbox, data), f => callback(f));
_client.Request(FrameFactory.Create(inbox, data), f => callback(f));
return true;
}
catch (Exception ex)
{
@ -100,7 +106,8 @@ namespace ZeroLevel.Network
{
try
{
return _client.Request(FrameFactory.Create(inbox), f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
_client.Request(FrameFactory.Create(inbox), f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
return true;
}
catch (Exception ex)
{
@ -113,7 +120,8 @@ namespace ZeroLevel.Network
{
try
{
return _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_INBOX), f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
_client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_INBOX), f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
return true;
}
catch (Exception ex)
{
@ -126,8 +134,9 @@ namespace ZeroLevel.Network
{
try
{
return _client.Request(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible<Trequest>(request)),
_client.Request(FrameFactory.Create(inbox, MessageSerializer.SerializeCompatible<Trequest>(request)),
f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
return true;
}
catch (Exception ex)
{
@ -140,8 +149,9 @@ namespace ZeroLevel.Network
{
try
{
return _client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_WITHOUT_ARGS_INBOX, MessageSerializer.SerializeCompatible<Trequest>(request)),
_client.Request(FrameFactory.Create(BaseSocket.DEFAULT_REQUEST_WITHOUT_ARGS_INBOX, MessageSerializer.SerializeCompatible<Trequest>(request)),
f => callback(MessageSerializer.DeserializeCompatible<Tresponse>(f)));
return true;
}
catch (Exception ex)
{

@ -1,7 +1,6 @@
using MemoryPools;
using System;
using System.Threading;
using ZeroLevel.Services.Pools;
namespace ZeroLevel.Network
{

@ -1,4 +1,5 @@
using System;
using MemoryPools;
using System;
using System.Collections.Concurrent;
using System.Net;
using System.Net.Sockets;
@ -19,8 +20,17 @@ namespace ZeroLevel.Network
public int identity;
public byte[] data;
}
private struct OutcomingFrame
{
public bool is_request;
public int identity;
public byte[] data;
}
private BlockingCollection<IncomingFrame> _incoming_queue = new BlockingCollection<IncomingFrame>();
private readonly JetValPool<OutcomingFrame> _outcomingFramesPool = new JetValPool<OutcomingFrame>();
private ConcurrentQueue<IncomingFrame> _incoming_queue = new ConcurrentQueue<IncomingFrame>();
private ConcurrentQueue<OutcomingFrame> _outcoming_queue = new ConcurrentQueue<OutcomingFrame>();
private ManualResetEventSlim _outcomingFrameEvent = new ManualResetEventSlim(false);
#endregion
private Socket _clientSocket;
@ -31,6 +41,7 @@ namespace ZeroLevel.Network
private readonly object _reconnection_lock = new object();
private long _heartbeat_key;
private Thread _receiveThread;
private Thread _sendingThread;
#endregion Private
@ -41,7 +52,6 @@ namespace ZeroLevel.Network
try
{
_clientSocket = new Socket(ep.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
_clientSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontLinger, true);
_clientSocket.Connect(ep);
OnConnect(this);
}
@ -81,6 +91,11 @@ namespace ZeroLevel.Network
_receiveThread.IsBackground = true;
_receiveThread.Start();
_sendingThread = new Thread(OutcomingFramesJob);
_sendingThread.IsBackground = true;
_sendingThread.Start();
_heartbeat_key = Sheduller.RemindEvery(TimeSpan.FromMilliseconds(MINIMUM_HEARTBEAT_UPDATE_PERIOD_MS), Heartbeat);
}
@ -109,27 +124,26 @@ namespace ZeroLevel.Network
public event Action<ISocketClient> OnDisconnect = (_) => { };
public IPEndPoint Endpoint { get; }
public bool Request(Frame frame, Action<byte[]> callback, Action<string> fail = null)
public void Request(Frame frame, Action<byte[]> callback, Action<string> fail = null)
{
if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Request] Socket status: {Status}");
if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Request] Socket status: {Status}");
var data = NetworkPacketFactory.Reqeust(MessageSerializer.Serialize(frame), out int id);
_requests.RegisterForFrame(id, callback, fail);
return Send(id, true, data);
Send(id, true, data);
}
public bool Send(Frame frame)
public void Send(Frame frame)
{
if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Send] Socket status: {Status}");
var data = NetworkPacketFactory.Message(MessageSerializer.Serialize(frame));
return Send(0, false, data);
Send(0, false, data);
}
public bool Response(byte[] data, int identity)
public void Response(byte[] data, int identity)
{
if (data == null) throw new ArgumentNullException(nameof(data));
if (Status != SocketClientStatus.Working) throw new Exception($"[SocketClient.Response] Socket status: {Status}");
return Send(0, false, NetworkPacketFactory.Response(data, identity));
Send(0, false, NetworkPacketFactory.Response(data, identity));
}
#endregion
@ -140,7 +154,7 @@ namespace ZeroLevel.Network
try
{
if (type == FrameType.KeepAlive) return;
_incoming_queue.Add(new IncomingFrame
_incoming_queue.Enqueue(new IncomingFrame
{
data = data,
type = type,
@ -223,75 +237,94 @@ namespace ZeroLevel.Network
IncomingFrame frame = default(IncomingFrame);
while (Status != SocketClientStatus.Disposed)
{
try
if (_incoming_queue.TryDequeue(out frame))
{
if (_incoming_queue.TryTake(out frame, 100))
try
{
try
switch (frame.type)
{
switch (frame.type)
{
case FrameType.Message:
Router?.HandleMessage(MessageSerializer.Deserialize<Frame>(frame.data), this);
break;
case FrameType.Request:
case FrameType.Message:
Router?.HandleMessage(MessageSerializer.Deserialize<Frame>(frame.data), this);
break;
case FrameType.Request:
{
Router?.HandleRequest(MessageSerializer.Deserialize<Frame>(frame.data), this, frame.identity, (id, response) =>
{
Router?.HandleRequest(MessageSerializer.Deserialize<Frame>(frame.data), this, frame.identity, (id, response) =>
if (response != null)
{
if (response != null)
{
this.Response(response, id);
}
});
}
break;
case FrameType.Response:
{
_requests.Success(frame.identity, frame.data);
}
break;
}
}
catch (Exception ex)
{
Log.SystemError(ex, "[SocketClient.IncomingFramesJob] Handle frame");
this.Response(response, id);
}
});
}
break;
case FrameType.Response:
{
_requests.Success(frame.identity, frame.data);
}
break;
}
}
}
catch (Exception ex)
{
Log.SystemError(ex, "[SocketClient.IncomingFramesJob] _incoming_queue.Take");
if (Status != SocketClientStatus.Disposed)
catch (Exception ex)
{
_incoming_queue.Dispose();
_incoming_queue = new BlockingCollection<IncomingFrame>();
Log.SystemError(ex, "[SocketClient.IncomingFramesJob] Handle frame");
}
continue;
}
else
{
Thread.Sleep(100);
}
}
}
private bool Send(int id, bool is_request, byte[] data)
private void OutcomingFramesJob()
{
if (Status == SocketClientStatus.Working)
while (Status != SocketClientStatus.Disposed)
{
try
if (Status == SocketClientStatus.Working)
{
if (is_request)
{
_requests.StartSend(id);
if (_outcomingFrameEvent.Wait(100))
{
_outcomingFrameEvent.Reset();
}
var sended = _clientSocket.Send(data, data.Length, SocketFlags.None);
return sended == data.Length;
while (_outcoming_queue.TryDequeue(out var frame))
{
try
{
if (frame.is_request)
{
_requests.StartSend(frame.identity);
}
_clientSocket.Send(frame.data, frame.data.Length, SocketFlags.None);
//var sended = _clientSocket.Send(frame.data, frame.data.Length, SocketFlags.None);
//return sended == frame.data.Length;
}
catch (Exception ex)
{
Log.SystemError(ex, $"[SocketClient.OutcomingFramesJob] _str_clientSocketeam.Send");
Broken();
OnDisconnect(this);
}
finally
{
_outcomingFramesPool.Return(frame);
}
}
}
catch (Exception ex)
else
{
Log.SystemError(ex, $"[SocketClient.SendFramesJob] _str_clientSocketeam.Send");
Broken();
OnDisconnect(this);
Thread.Sleep(400);
}
}
return false;
}
private void Send(int id, bool is_request, byte[] data)
{
var frame = _outcomingFramesPool.Get();
frame.data = data;
frame.identity = id;
frame.is_request = is_request;
_outcoming_queue.Enqueue(frame);
_outcomingFrameEvent.Set();
}
#endregion

@ -1,5 +1,4 @@
using System.Collections.Generic;
using ZeroLevel.Services.Semantic;
using ZeroLevel.Services.Serialization;
namespace ZeroLevel.Services.Semantic

@ -44,5 +44,41 @@ namespace ZeroLevel.Services.Semantic
_pool.Return(buffer);
}
}
public static IEnumerable<string> TokenizeCaseSensitive(string text)
{
int index = 0;
bool first = true;
var buffer = _pool.Rent(ARRAY_SIZE);
try
{
for (int i = 0; i < text?.Length; i++)
{
if (first && Char.IsLetter(text[i]))
{
first = false;
buffer[index++] = text[i];
}
else if (first == false && Char.IsLetterOrDigit(text[i]))
{
buffer[index++] = text[i];
}
else if (index > 0)
{
yield return new string(buffer, 0, index);
index = 0;
first = true;
}
}
if (index > 0)
{
yield return new string(buffer, 0, index);
}
}
finally
{
_pool.Return(buffer);
}
}
}
}

@ -0,0 +1,159 @@
using System.Collections.Generic;
using System.Linq;
namespace ZeroLevel.Services.Text
{
internal struct state
{
public int len;
public int link;
public Dictionary<char, int> next;
}
public class SuffixAutomata
{
const int MAXLEN = 100000;
private state[] st = new state[MAXLEN * 2];
int sz, last;
public void Init(bool is_reused = false)
{
sz = last = 0;
st[0].len = 0;
st[0].link = -1;
++sz;
// этот код нужен, только если автомат строится много раз для разных строк:
for (int i = 0; i < MAXLEN * 2; ++i)
{
st[i].next = new Dictionary<char, int>();
}
}
public void Extend(char c)
{
int cur = sz++;
st[cur].len = st[last].len + 1;
int p;
for (p = last; p != -1 && !st[p].next.ContainsKey(c); p = st[p].link)
st[p].next[c] = cur;
if (p == -1)
st[cur].link = 0;
else
{
int q = st[p].next[c];
if (st[p].len + 1 == st[q].len)
st[cur].link = q;
else
{
int clone = sz++;
st[clone].len = st[p].len + 1;
st[clone].next = st[q].next;
st[clone].link = st[q].link;
for (; p != -1 && st[p].next.ContainsKey(c) && st[p].next[c] == q; p = st[p].link)
st[p].next[c] = clone;
st[q].link = st[cur].link = clone;
}
}
last = cur;
}
public bool IsSubstring(string w)
{
if (string.IsNullOrEmpty(w)) return true;
bool fail = false;
int n, si = 0;
for (; si < last; si++)
{
if (st[si].next.ContainsKey(w[0]))
{
var start = st[si];
for (int i = 0; i < w.Length; i++)
{
if (start.next.ContainsKey(w[i]) == false)
{
fail = true;
break;
}
n = start.next[w[i]];
start = st[n];
}
break;
}
}
if (si == last)
{
fail = true;
}
return (!fail);
}
public string Intersection(string t)
{
var entries = st.Where(x => x.next.ContainsKey(t[0])).ToArray();
var candidates = entries
.Select(s => Intersection(s, t));
if (candidates != null && candidates.Any())
{
var max = candidates.Max(s => s?.Length ?? 0);
return candidates.FirstOrDefault(c => c != null && c.Length == max);
}
return null;
/*
int v = 0, l = 0, best = 0, bestpos = 0;
for (int i = 0; i < (int)t.Length; ++i)
{
while (v > 0 && !st[v].next.ContainsKey(t[i]))
{
v = st[v].link;
l = st[v].len;
}
if (st[v].next.ContainsKey(t[i]))
{
v = st[v].next[t[i]];
++l;
}
if (l > best)
{
best = l;
bestpos = i;
}
}
var start = bestpos - best + 1;
var length = best;
if (start >= 0 && start < t.Length && (start + length) <= t.Length)
return t.Substring(start, length);
return null;
*/
}
private string Intersection(state entry, string t)
{
int v = 0, l = 0, best = 0, bestpos = 0;
for (int i = 0; i < (int)t.Length; ++i)
{
while (v > 0 && !entry.next.ContainsKey(t[i]))
{
v = entry.link;
l = entry.len;
entry = st[v];
}
if (entry.next.ContainsKey(t[i]))
{
v = entry.next[t[i]];
entry = st[v];
++l;
}
if (l > best)
{
best = l;
bestpos = i;
}
}
var start = bestpos - best + 1;
var length = best;
if (start >= 0 && start < t.Length && (start + length) <= t.Length)
return t.Substring(start, length);
return null;
}
}
}

@ -39,9 +39,13 @@
public void SkipBreaks()
{
while (EOF == false && char.IsWhiteSpace(Current)) Move();
while (EOF == false && (Current == '\n' || Current == '\r')) Move();
}
public bool IsBrackets() => EOF == false && (Current == '\'' || Current == '"' || Current == '”');
public bool IsBreak() => EOF == false && (Current == '\n' || Current == '\r');
public bool IsWhiteSpace() => EOF == false && char.IsWhiteSpace(Current);
public bool MoveBack()
{
_position = _position - 1;
@ -84,7 +88,7 @@
index++;
identity = _template.Substring(offset, index - offset);
}
return identity.ToLowerInvariant();
return identity;
}
public string ReadWord()

@ -22,7 +22,7 @@ namespace ZeroLevel.Services
output = Regex.Replace(output, @"\s|\.|\(", " ");
output = Regex.Replace(output, @"\s+", " ");
output = Regex.Replace(output, @"[^\s\w\d-]", "");
//output = Regex.Replace(output, @"[^\s\w\d-]", "");
output = output.Trim();
Dictionary<string, string> tdict = GetDictionaryByType(type);
@ -145,7 +145,7 @@ namespace ZeroLevel.Services
gost.Add("«", "");
gost.Add("»", "");
gost.Add("—", "-");
gost.Add(" ", "-");
gost.Add(" ", " ");
iso.Add("Є", "YE");
iso.Add("І", "I");
@ -223,7 +223,7 @@ namespace ZeroLevel.Services
iso.Add("«", "");
iso.Add("»", "");
iso.Add("—", "-");
iso.Add(" ", "-");
iso.Add(" ", " ");
}
}
}

@ -346,7 +346,7 @@ namespace ZeroLevel.Services.Web
if (cursor.Current == '&')
{
cursor.Move();
var identity = cursor.ReadIdentity();
var identity = cursor.ReadIdentity().ToLowerInvariant();
cursor.MoveBack();
if (_entityMap.ContainsKey(identity + ";"))
{

@ -6,7 +6,7 @@
</Description>
<Authors>ogoun</Authors>
<Company>ogoun</Company>
<AssemblyVersion>3.3.6.3</AssemblyVersion>
<AssemblyVersion>3.3.6.4</AssemblyVersion>
<PackageReleaseNotes>Configuration floating number convert fix</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2022</Copyright>
@ -14,8 +14,8 @@
<PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<Version>3.3.6.3</Version>
<FileVersion>3.3.6.3</FileVersion>
<Version>3.3.6.4</Version>
<FileVersion>3.3.6.4</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType>

Loading…
Cancel
Save

Powered by TurnKey Linux.