From 63eb20dcef3f7ab6dedb34f39f3d5debd7036828 Mon Sep 17 00:00:00 2001 From: "a.bozhenov" Date: Wed, 5 Jun 2019 16:22:58 +0300 Subject: [PATCH] Fixes --- TestApp/Program.cs | 3 +- TestApp/TestApp.csproj | 6 ++ TestApp/config.ini | 14 +++ ZeroLevel/Models/InvokeResult.cs | 3 + .../Services/Config/BaseConfiguration.cs | 4 +- ZeroLevel/Services/Config/Configuration.cs | 1 + ZeroLevel/Services/Config/IConfiguration.cs | 4 + .../Logging/Implementation/TextFileLogger.cs | 58 +++++------ .../Services/Network/Contract/IExClient.cs | 2 - .../Services/Network/Contract/IZBackward.cs | 7 +- .../Network/FileTransfer/FileClient.cs | 6 +- .../Network/FileTransfer/FileReceiver.cs | 20 +++- .../Network/FileTransfer/FileServer.cs | 53 ++-------- .../Services/Network/Services/ExClient.cs | 13 +-- .../Network/Services/RequestBuffer.cs | 98 +++++++++++++++++++ ZeroLevel/Services/Network/ZBaseNetwork.cs | 2 +- ZeroLevel/Services/Network/ZSocketClient.cs | 93 +----------------- .../Services/Network/ZSocketServerClient.cs | 49 +++++----- ZeroLevel/ZeroLevel.csproj | 6 +- 19 files changed, 225 insertions(+), 217 deletions(-) create mode 100644 TestApp/config.ini create mode 100644 ZeroLevel/Services/Network/Services/RequestBuffer.cs diff --git a/TestApp/Program.cs b/TestApp/Program.cs index 663a17b..ada05d7 100644 --- a/TestApp/Program.cs +++ b/TestApp/Program.cs @@ -6,8 +6,7 @@ namespace TestApp { private static void Main(string[] args) { - Log.AddConsoleLogger(); - Bootstrap.Startup(args); + Bootstrap.Startup(args, () => Configuration.ReadSetFromIniFile("config.ini")); } } } diff --git a/TestApp/TestApp.csproj b/TestApp/TestApp.csproj index 3a4521e..c638033 100644 --- a/TestApp/TestApp.csproj +++ b/TestApp/TestApp.csproj @@ -9,4 +9,10 @@ + + + Always + + + diff --git a/TestApp/config.ini b/TestApp/config.ini new file mode 100644 index 0000000..6382ac9 --- /dev/null +++ b/TestApp/config.ini @@ -0,0 +1,14 @@ +discovery=127.0.0.1:5012 + +[service] +AppName=TestApp +AppKey=test.app +ServiceGroup=seo +ServiceType=site + +[log] +path=logs +console=true + +[webapi] +port=8183 diff --git a/ZeroLevel/Models/InvokeResult.cs b/ZeroLevel/Models/InvokeResult.cs index a063ce2..2b61570 100644 --- a/ZeroLevel/Models/InvokeResult.cs +++ b/ZeroLevel/Models/InvokeResult.cs @@ -14,6 +14,7 @@ namespace ZeroLevel.Models #region Static private static readonly InvokeResult _successResultWitoutComment = new InvokeResult(true, String.Empty); + private static readonly InvokeResult _faultResultWitoutComment = new InvokeResult(false, String.Empty); #endregion Static @@ -54,6 +55,8 @@ namespace ZeroLevel.Models /// public static InvokeResult Fault(string comment) { return new InvokeResult(false, comment); } + public static InvokeResult Fault() { return _faultResultWitoutComment; } + /// /// Successfully /// diff --git a/ZeroLevel/Services/Config/BaseConfiguration.cs b/ZeroLevel/Services/Config/BaseConfiguration.cs index 56a9fc5..d8f81fc 100644 --- a/ZeroLevel/Services/Config/BaseConfiguration.cs +++ b/ZeroLevel/Services/Config/BaseConfiguration.cs @@ -454,9 +454,9 @@ namespace ZeroLevel.Services.Config public void CopyTo(IConfiguration config) { - foreach (var key in config.Keys) + foreach (var key in this.Keys) { - this.Append(key, config[key]); + config.Append(key, this[key]); } } } diff --git a/ZeroLevel/Services/Config/Configuration.cs b/ZeroLevel/Services/Config/Configuration.cs index 310b658..02e2513 100644 --- a/ZeroLevel/Services/Config/Configuration.cs +++ b/ZeroLevel/Services/Config/Configuration.cs @@ -30,6 +30,7 @@ namespace ZeroLevel _emptySet = new BaseConfigurationSet(); _empty.Freeze(true); _emptySet.FreezeConfiguration(true); + DefaultSet = Configuration.CreateSet(); } #endregion Ctor diff --git a/ZeroLevel/Services/Config/IConfiguration.cs b/ZeroLevel/Services/Config/IConfiguration.cs index c5b3985..7f3b27d 100644 --- a/ZeroLevel/Services/Config/IConfiguration.cs +++ b/ZeroLevel/Services/Config/IConfiguration.cs @@ -118,6 +118,10 @@ namespace ZeroLevel /// IConfiguration Append(string key, string value); + /// + /// Append key and values list + /// + IConfiguration Append(string key, IEnumerable values); /// /// Set key with one value, if any values by key exists, they will be dropped /// diff --git a/ZeroLevel/Services/Logging/Implementation/TextFileLogger.cs b/ZeroLevel/Services/Logging/Implementation/TextFileLogger.cs index 692dc0e..d835757 100644 --- a/ZeroLevel/Services/Logging/Implementation/TextFileLogger.cs +++ b/ZeroLevel/Services/Logging/Implementation/TextFileLogger.cs @@ -98,43 +98,39 @@ namespace ZeroLevel.Services.Logging.Implementation internal static TextFileLoggerOptions CreateOptionsBy(IConfiguration config, string path, string logPrefix) { - if (config.Contains(logPrefix)) - { - var options = new TextFileLoggerOptions(). - SetFolderPath(path); + var options = new TextFileLoggerOptions(). + SetFolderPath(path); - config.DoWithFirst($"{logPrefix}backlog", backlog => + config.DoWithFirst($"{logPrefix}backlog", backlog => + { + if (backlog > 0) { - if (backlog > 0) - { - Log.Backlog(backlog); - } - }); - config.DoWithFirst($"{logPrefix}archive", enable => + Log.Backlog(backlog); + } + }); + config.DoWithFirst($"{logPrefix}archive", enable => + { + if (enable) { - if (enable) - { - options.EnableAutoArchiving(); - } - }); - config.DoWithFirst($"{logPrefix}sizeinkb", size => + options.EnableAutoArchiving(); + } + }); + config.DoWithFirst($"{logPrefix}sizeinkb", size => + { + if (size >= 1) { - if (size >= 1) - { - options.SetMaximumFileSizeInKb(size); - } - }); + options.SetMaximumFileSizeInKb(size); + } + }); - config.DoWithFirst($"{logPrefix}cleanolderdays", days => + config.DoWithFirst($"{logPrefix}cleanolderdays", days => + { + if (days > 0) { - if (days > 0) - { - options.EnableAutoCleaning(TimeSpan.FromDays(days)); - } - }); - return options; - } - return null; + options.EnableAutoCleaning(TimeSpan.FromDays(days)); + } + }); + return options; } } diff --git a/ZeroLevel/Services/Network/Contract/IExClient.cs b/ZeroLevel/Services/Network/Contract/IExClient.cs index 99525c8..183d711 100644 --- a/ZeroLevel/Services/Network/Contract/IExClient.cs +++ b/ZeroLevel/Services/Network/Contract/IExClient.cs @@ -30,7 +30,5 @@ namespace ZeroLevel.Network void RegisterInbox(string inbox, Action handler); void RegisterInbox(Action handler); - - void RegisterInbox(string inbox, Func handler); } } \ No newline at end of file diff --git a/ZeroLevel/Services/Network/Contract/IZBackward.cs b/ZeroLevel/Services/Network/Contract/IZBackward.cs index cc6e3e3..6271016 100644 --- a/ZeroLevel/Services/Network/Contract/IZBackward.cs +++ b/ZeroLevel/Services/Network/Contract/IZBackward.cs @@ -1,5 +1,4 @@ -using System; -using System.Net; +using System.Net; using ZeroLevel.Models; namespace ZeroLevel.Network @@ -8,7 +7,7 @@ namespace ZeroLevel.Network { IPEndPoint Endpoint { get; } - void SendBackward(Frame frame); - void SendBackward(string inbox, T message); + InvokeResult SendBackward(Frame frame); + InvokeResult SendBackward(string inbox, T message); } } \ No newline at end of file diff --git a/ZeroLevel/Services/Network/FileTransfer/FileClient.cs b/ZeroLevel/Services/Network/FileTransfer/FileClient.cs index d47b9e9..81be498 100644 --- a/ZeroLevel/Services/Network/FileTransfer/FileClient.cs +++ b/ZeroLevel/Services/Network/FileTransfer/FileClient.cs @@ -23,9 +23,9 @@ namespace ZeroLevel.Services.Network.FileTransfer _nameMapper = nameMapper ?? throw new Exception(nameof(nameMapper)); _disposeClient = disposeClient; - _client.RegisterInbox("__upload_file_start", (f, _, __) => Receiver.Incoming(f, nameMapper(_client))); - _client.RegisterInbox("__upload_file_frame", (f, _, __) => Receiver.Incoming(f)); - _client.RegisterInbox("__upload_file_complete", (f, _, __) => Receiver.Incoming(f)); + _client.RegisterInbox("__upload_file_start", (f, _, __) => Receiver.Incoming(f, nameMapper(_client))); + _client.RegisterInbox("__upload_file_frame", (f, _, __) => Receiver.Incoming(f)); + _client.RegisterInbox("__upload_file_complete", (f, _, __) => Receiver.Incoming(f)); } public void Dispose() diff --git a/ZeroLevel/Services/Network/FileTransfer/FileReceiver.cs b/ZeroLevel/Services/Network/FileTransfer/FileReceiver.cs index 0e13a1f..da3653c 100644 --- a/ZeroLevel/Services/Network/FileTransfer/FileReceiver.cs +++ b/ZeroLevel/Services/Network/FileTransfer/FileReceiver.cs @@ -13,6 +13,18 @@ namespace ZeroLevel.Services.Network.FileTransfer { private readonly FileStream _stream; internal DateTime _writeTime { get; private set; } = DateTime.UtcNow; + private bool _gotCompleteMessage = false; + + public bool GotCompleteMessage() => _gotCompleteMessage = true; + + public bool ReadyToRemove() + { + if (_gotCompleteMessage) + { + return (DateTime.UtcNow - _writeTime).TotalSeconds > 15; + } + return false; + } public FileWriter(string path) { @@ -59,7 +71,7 @@ namespace ZeroLevel.Services.Network.FileTransfer { foreach (var pair in _incoming) { - if (pair.Value.IsTimeoutBy(TimeSpan.FromMinutes(3))) + if (pair.Value.IsTimeoutBy(TimeSpan.FromMinutes(3)) || pair.Value.ReadyToRemove()) { Remove(pair.Key); } @@ -116,7 +128,11 @@ namespace ZeroLevel.Services.Network.FileTransfer { lock (_locker) { - Remove(info.UploadFileTaskId); + FileWriter stream; + if (_incoming.TryGetValue(info.UploadFileTaskId, out stream)) + { + stream.GotCompleteMessage(); + } } } catch (Exception ex) diff --git a/ZeroLevel/Services/Network/FileTransfer/FileServer.cs b/ZeroLevel/Services/Network/FileTransfer/FileServer.cs index 897ba5d..3a39a0c 100644 --- a/ZeroLevel/Services/Network/FileTransfer/FileServer.cs +++ b/ZeroLevel/Services/Network/FileTransfer/FileServer.cs @@ -1,6 +1,4 @@ using System; -using System.IO; -using System.Threading; using ZeroLevel.Models; using ZeroLevel.Network; using ZeroLevel.Services.Network.FileTransfer.Model; @@ -43,54 +41,21 @@ namespace ZeroLevel.Services.Network.FileTransfer internal override void ExecuteSendFile(FileReader reader, FileTransferTask task) { - - /* Log.Info($"Start upload file {reader.Path}"); var startinfo = reader.GetStartInfo(); - using (var signal = new ManualResetEvent(false)) + if (false == task.Client.SendBackward("__upload_file_start", startinfo).Success) { - bool next = false; - - if (false == task.Client.RequestBackward("__upload_file_start", startinfo, - r => - { - next = r.Success; - signal.Set(); - }).Success) - { - next = false; - signal.Set(); - } - signal.WaitOne(5000); - if (next) - { - foreach (var chunk in reader.Read()) - { - signal.Reset(); - if (task.Client.RequestBackward("__upload_file_frame", chunk, r => next = r.Success).Success == false) - { - next = false; - signal.Set(); - } - signal.WaitOne(); - if (!next) - { - break; - } - } - } - if (next) + return; + } + foreach (var chunk in reader.Read()) + { + if (task.Client.SendBackward("__upload_file_frame", chunk).Success == false) { - task.Client.RequestBackward("__upload_file_complete", reader.GetCompleteInfo(), r => - { - if (r.Success == false) - { - Log.Warning($"Unsuccess send file. {r.Comment}"); - } - }); + return; } } - Log.Debug($"Stop upload file {reader.Path}");*/ + task.Client.SendBackward("__upload_file_complete", reader.GetCompleteInfo()); + Log.Debug($"Stop upload file {reader.Path}"); } } } diff --git a/ZeroLevel/Services/Network/Services/ExClient.cs b/ZeroLevel/Services/Network/Services/ExClient.cs index ab40a6b..ac665ec 100644 --- a/ZeroLevel/Services/Network/Services/ExClient.cs +++ b/ZeroLevel/Services/Network/Services/ExClient.cs @@ -71,11 +71,6 @@ namespace ZeroLevel.Network _router.RegisterInbox(DEFAULT_MESSAGE_INBOX, handler); } - public void RegisterInbox(string inbox, Func handler) - { - _router.RegisterInbox(inbox, handler); - } - public InvokeResult Request(Action callback) { return _fe.Request(DEFAULT_REQUEST_INBOX, resp => callback(resp)); @@ -106,14 +101,14 @@ namespace ZeroLevel.Network return _fe.Send(inbox, obj); } - public void SendBackward(Frame frame) + public InvokeResult SendBackward(Frame frame) { - _fe.Send(frame); + return _fe.Send(frame); } - public void SendBackward(string inbox, T obj) + public InvokeResult SendBackward(string inbox, T obj) { - Send(inbox, obj); + return Send(inbox, obj); } } } \ No newline at end of file diff --git a/ZeroLevel/Services/Network/Services/RequestBuffer.cs b/ZeroLevel/Services/Network/Services/RequestBuffer.cs new file mode 100644 index 0000000..34aa109 --- /dev/null +++ b/ZeroLevel/Services/Network/Services/RequestBuffer.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using ZeroLevel.Network; +using ZeroLevel.Services.Pools; + +namespace ZeroLevel.Network +{ + internal sealed class RequestBuffer + { + private readonly object _reqeust_lock = new object(); + private Dictionary _requests = new Dictionary(); + private static ObjectPool _ri_pool = new ObjectPool(() => new RequestInfo()); + + public void RegisterForFrame(Frame frame, Action callback, Action fail = null) + { + var ri = _ri_pool.Allocate(); + lock (_reqeust_lock) + { + ri.Reset(callback, fail); + _requests.Add(frame.FrameId, ri); + } + } + + public void Fail(long frameId, string message) + { + RequestInfo ri = null; + lock (_reqeust_lock) + { + if (_requests.ContainsKey(frameId)) + { + ri = _requests[frameId]; + _requests.Remove(frameId); + } + } + if (ri != null) + { + ri.Fail(message); + _ri_pool.Free(ri); + } + } + + public void Success(long frameId, Frame frame) + { + RequestInfo ri = null; + lock (_reqeust_lock) + { + if (_requests.ContainsKey(frameId)) + { + ri = _requests[frameId]; + _requests.Remove(frameId); + } + } + if (ri != null) + { + ri.Success(frame); + _ri_pool.Free(ri); + } + } + + public void StartSend(long frameId) + { + RequestInfo ri = null; + lock (_reqeust_lock) + { + if (_requests.ContainsKey(frameId)) + { + ri = _requests[frameId]; + } + } + if (ri != null) + { + ri.StartSend(); + } + } + + public void TestForTimeouts() + { + var now_ticks = DateTime.UtcNow.Ticks; + var to_remove = new List(); + lock (_reqeust_lock) + { + foreach (var pair in _requests) + { + if (pair.Value.Sended == false) continue; + var diff = now_ticks - pair.Value.Timestamp; + if (diff > ZBaseNetwork.MAX_REQUEST_TIME_TICKS) + { + to_remove.Add(pair.Key); + } + } + } + foreach (var key in to_remove) + { + Fail(key, "Timeout"); + } + } + } +} diff --git a/ZeroLevel/Services/Network/ZBaseNetwork.cs b/ZeroLevel/Services/Network/ZBaseNetwork.cs index aaa98c7..840f926 100644 --- a/ZeroLevel/Services/Network/ZBaseNetwork.cs +++ b/ZeroLevel/Services/Network/ZBaseNetwork.cs @@ -29,7 +29,7 @@ namespace ZeroLevel.Network /// /// The period of the request, after which it is considered unsuccessful /// - protected const long MAX_REQUEST_TIME_TICKS = 30000 * TimeSpan.TicksPerMillisecond; + internal const long MAX_REQUEST_TIME_TICKS = 30000 * TimeSpan.TicksPerMillisecond; public const int MAX_REQUEST_TIME_MS = 30000; diff --git a/ZeroLevel/Services/Network/ZSocketClient.cs b/ZeroLevel/Services/Network/ZSocketClient.cs index 82857f4..f9914a4 100644 --- a/ZeroLevel/Services/Network/ZSocketClient.cs +++ b/ZeroLevel/Services/Network/ZSocketClient.cs @@ -1,10 +1,8 @@ using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.Net; using System.Net.Sockets; using System.Threading; -using ZeroLevel.Services.Pools; using ZeroLevel.Services.Serialization; namespace ZeroLevel.Network @@ -12,96 +10,7 @@ namespace ZeroLevel.Network public class ZSocketClient : ZBaseNetwork, IZTransport { - private class RequestBuffer - { - private readonly object _reqeust_lock = new object(); - private Dictionary _requests = new Dictionary(); - private static ObjectPool _ri_pool = new ObjectPool(() => new RequestInfo()); - - public void RegisterForFrame(Frame frame, Action callback, Action fail = null) - { - var ri = _ri_pool.Allocate(); - lock (_reqeust_lock) - { - ri.Reset(callback, fail); - _requests.Add(frame.FrameId, ri); - } - } - - public void Fail(long frameId, string message) - { - RequestInfo ri = null; - lock (_reqeust_lock) - { - if (_requests.ContainsKey(frameId)) - { - ri = _requests[frameId]; - _requests.Remove(frameId); - } - } - if (ri != null) - { - ri.Fail(message); - _ri_pool.Free(ri); - } - } - - public void Success(long frameId, Frame frame) - { - RequestInfo ri = null; - lock (_reqeust_lock) - { - if (_requests.ContainsKey(frameId)) - { - ri = _requests[frameId]; - _requests.Remove(frameId); - } - } - if (ri != null) - { - ri.Success(frame); - _ri_pool.Free(ri); - } - } - - public void StartSend(long frameId) - { - RequestInfo ri = null; - lock (_reqeust_lock) - { - if (_requests.ContainsKey(frameId)) - { - ri = _requests[frameId]; - } - } - if (ri != null) - { - ri.StartSend(); - } - } - - public void TestForTimeouts() - { - var now_ticks = DateTime.UtcNow.Ticks; - var to_remove = new List(); - lock (_reqeust_lock) - { - foreach (var pair in _requests) - { - if (pair.Value.Sended == false) continue; - var diff = now_ticks - pair.Value.Timestamp; - if (diff > ZBaseNetwork.MAX_REQUEST_TIME_TICKS) - { - to_remove.Add(pair.Key); - } - } - } - foreach (var key in to_remove) - { - Fail(key, "Timeout"); - } - } - } + #region Private diff --git a/ZeroLevel/Services/Network/ZSocketServerClient.cs b/ZeroLevel/Services/Network/ZSocketServerClient.cs index 3380d4e..01a51fd 100644 --- a/ZeroLevel/Services/Network/ZSocketServerClient.cs +++ b/ZeroLevel/Services/Network/ZSocketServerClient.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Net; using System.Net.Sockets; using System.Threading; +using ZeroLevel.Models; using ZeroLevel.Services.Serialization; namespace ZeroLevel.Network @@ -52,7 +53,7 @@ namespace ZeroLevel.Network _stream.BeginRead(_buffer, 0, DEFAULT_RECEIVE_BUFFER_SIZE, ReceiveAsyncCallback, null); } - public void SendBackward(Frame frame) + public InvokeResult SendBackward(Frame frame) { if (frame != null && Status == ZTransportStatus.Working && false == _send_queue.IsCompleted && false == _send_queue.IsAddingCompleted) { @@ -60,6 +61,7 @@ namespace ZeroLevel.Network try { _send_queue.Add(NetworkStreamFastObfuscator.PrepareData(data)); + return InvokeResult.Succeeding(); } catch (ObjectDisposedException) { @@ -70,6 +72,30 @@ namespace ZeroLevel.Network frame?.Release(); } } + return InvokeResult.Fault(); + } + + public InvokeResult SendBackward(string inbox, T message) + { + var frame = FrameBuilder.BuildFrame(message, inbox); + if (Status == ZTransportStatus.Working && false == _send_queue.IsCompleted && false == _send_queue.IsAddingCompleted) + { + var data = MessageSerializer.Serialize(frame); + try + { + _send_queue.Add(NetworkStreamFastObfuscator.PrepareData(data)); + return InvokeResult.Succeeding(); + } + catch (ObjectDisposedException) + { + // Ignore + } + finally + { + frame?.Release(); + } + } + return InvokeResult.Fault(); } private void SendFramesJob() @@ -205,26 +231,5 @@ namespace ZeroLevel.Network if (other == null) return false; return this.Endpoint.Compare(other.Endpoint) == 0; } - - public void SendBackward(string inbox, T message) - { - var frame = FrameBuilder.BuildFrame(message, inbox); - if (Status == ZTransportStatus.Working && false == _send_queue.IsCompleted && false == _send_queue.IsAddingCompleted) - { - var data = MessageSerializer.Serialize(frame); - try - { - _send_queue.Add(NetworkStreamFastObfuscator.PrepareData(data)); - } - catch (ObjectDisposedException) - { - // Ignore - } - finally - { - frame?.Release(); - } - } - } } } \ No newline at end of file diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index f37b2c6..6c315e9 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -5,16 +5,16 @@ Infrastructure layer library ogoun ogoun - 2.0.6.0 + 2.0.7.0 Refactoring configuration and log -File client/server (no stable) +File client/server update(no stable) https://github.com/ogoun/Zero/wiki Copyright Ogoun 2019 https://opensource.org/licenses/MIT https://raw.githubusercontent.com/ogoun/Zero/master/zero.png https://github.com/ogoun/Zero GitHub - 2.0.6 + 2.0.7