diff --git a/PartitionFileStorageTest/Program.cs b/PartitionFileStorageTest/Program.cs index 783bb87..107d21d 100644 --- a/PartitionFileStorageTest/Program.cs +++ b/PartitionFileStorageTest/Program.cs @@ -76,7 +76,7 @@ namespace PartitionFileStorageTest static void Main(string[] args) { var testDict = new Dictionary>>(); - var options = new PartitionFileStorageOptions + var options = new PartitionFileSystemStorageOptions { MaxDegreeOfParallelism = 1, DataConverter = new DataConverter(), @@ -86,7 +86,7 @@ namespace PartitionFileStorageTest }; options.Partitions.Add(new Partition("data", p => p.Date.ToString("yyyyMMdd"))); options.Partitions.Add(new Partition("ctn", p => p.Ctn.ToString().PadLeft(COUNT_NUMBERS, '0'))); - var storage = new PartitionFileStorage(options); + var storage = new PartitionFileSystemStorage(options); for (int i = 0; i < 50000; i++) { diff --git a/ZeroLevel/Services/Storages/PartitionFileSystemStorage/IPartitionDataConverter.cs b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/IPartitionDataConverter.cs new file mode 100644 index 0000000..bf2fbb4 --- /dev/null +++ b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/IPartitionDataConverter.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.IO; + +namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage +{ + public interface IPartitionDataConverter + { + IEnumerable ReadFromStorage(Stream stream); + void WriteToStorage(Stream stream, IEnumerable data); + } +} diff --git a/ZeroLevel/Services/Storages/PartitionFileSystemStorage/IPartitionFileStorage.cs b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/IPartitionFileStorage.cs new file mode 100644 index 0000000..25adbfe --- /dev/null +++ b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/IPartitionFileStorage.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage +{ + public interface IPartitionFileStorage + { + Task WriteAsync(TKey key, IEnumerable records); + Task> CollectAsync(IEnumerable keys); + void Drop(TKey key); + } +} diff --git a/ZeroLevel/Services/Storages/PartitionFileSystemStorage/Partition.cs b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/Partition.cs new file mode 100644 index 0000000..4b11984 --- /dev/null +++ b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/Partition.cs @@ -0,0 +1,18 @@ +using System; + +namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage +{ + /// + /// Make part of full file path + /// + public class Partition + { + public Partition(string name, Func pathExtractor) + { + Name = name; + PathExtractor = pathExtractor; + } + public Func PathExtractor { get; } + public string Name { get; } + } +} diff --git a/ZeroLevel/Services/Storages/PartitionFileStorage.cs b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/PartitionFileSystemStorage.cs similarity index 72% rename from ZeroLevel/Services/Storages/PartitionFileStorage.cs rename to ZeroLevel/Services/Storages/PartitionFileSystemStorage/PartitionFileSystemStorage.cs index 1808626..f4c75fc 100644 --- a/ZeroLevel/Services/Storages/PartitionFileStorage.cs +++ b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/PartitionFileSystemStorage.cs @@ -7,51 +7,14 @@ using System.Linq; using System.Threading.Tasks; using ZeroLevel.Services.FileSystem; -namespace ZeroLevel.Services.Storages +namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage { - public class Partition - { - public Partition(string name, Func pathExtractor) - { - Name = name; - PathExtractor = pathExtractor; - } - public Func PathExtractor { get; } - public string Name { get; } - } - /// - /// Write data to partitional data storage - /// - public interface IPartitionFileStorage - { - Task WriteAsync(TKey key, IEnumerable records); - Task> CollectAsync(IEnumerable keys); - void Drop(TKey key); - } - - public interface IPartitionDataConverter - { - IEnumerable ConvertFromStorage(Stream stream); - void ConvertToStorage(Stream stream, IEnumerable data); - } - - public class PartitionFileStorageOptions - { - public string RootFolder { get; set; } - public int MaxDegreeOfParallelism { get; set; } = Environment.ProcessorCount / 2; - public bool MergeFiles { get; set; } = false; - public int MergeFrequencyInMinutes { get; set; } = 180; - public bool UseCompression { get; set; } = false; - public IPartitionDataConverter DataConverter { get; set; } - public List> Partitions { get; set; } = new List>(); - } - - public class PartitionFileStorage + public class PartitionFileSystemStorage : IPartitionFileStorage { - private readonly PartitionFileStorageOptions _options; - public PartitionFileStorage(PartitionFileStorageOptions options) + private readonly PartitionFileSystemStorageOptions _options; + public PartitionFileSystemStorage(PartitionFileSystemStorageOptions options) { if (options.RootFolder == null) throw new ArgumentNullException(nameof(options.RootFolder)); @@ -86,7 +49,7 @@ namespace ZeroLevel.Services.Storages { using (var stream = CreateReadStream(file)) { - foreach (var item in _options.DataConverter.ConvertFromStorage(stream)) + foreach (var item in _options.DataConverter.ReadFromStorage(stream)) { set.Add(item); } @@ -99,7 +62,7 @@ namespace ZeroLevel.Services.Storages { using (var stream = CreateWriteStream(key)) { - _options.DataConverter.ConvertToStorage(stream, records); + _options.DataConverter.WriteToStorage(stream, records); await stream.FlushAsync(); } } @@ -125,7 +88,7 @@ namespace ZeroLevel.Services.Storages var files = Directory.GetFiles(path); if (files != null && files.Length > 1) { - + // TODO } } @@ -171,7 +134,7 @@ namespace ZeroLevel.Services.Storages { var ms = new MemoryStream(); using (var compressed = new GZipStream(stream, CompressionMode.Decompress, false)) - { + { compressed.CopyTo(ms); } ms.Position = 0; diff --git a/ZeroLevel/Services/Storages/PartitionFileSystemStorage/PartitionFileSystemStorageOptions.cs b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/PartitionFileSystemStorageOptions.cs new file mode 100644 index 0000000..ac8ad5f --- /dev/null +++ b/ZeroLevel/Services/Storages/PartitionFileSystemStorage/PartitionFileSystemStorageOptions.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; + +namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage +{ + public class PartitionFileSystemStorageOptions + { + public string RootFolder { get; set; } + public int MaxDegreeOfParallelism { get; set; } = Environment.ProcessorCount / 2; + public bool MergeFiles { get; set; } = false; + public int MergeFrequencyInMinutes { get; set; } = 180; + public bool UseCompression { get; set; } = false; + public IPartitionDataConverter DataConverter { get; set; } + public List> Partitions { get; set; } = new List>(); + } +} diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index de209f9..1dbfa7c 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -6,17 +6,16 @@ ogoun ogoun - 3.3.6.6 - Configuration binding refactoring. -Append custom parsing attribute. + 3.3.6.7 + PartitionFileSystemStorage https://github.com/ogoun/Zero/wiki Copyright Ogoun 2022 https://github.com/ogoun/Zero git - 3.3.6.6 - 3.3.6.6 + 3.3.6.7 + 3.3.6.7 AnyCPU;x64;x86 zero.png full