PartitionFileSystemStorage

pull/3/head
Ogoun 2 years ago
parent 829d39d854
commit a996f36abe

@ -76,7 +76,7 @@ namespace PartitionFileStorageTest
static void Main(string[] args) static void Main(string[] args)
{ {
var testDict = new Dictionary<ulong, Dictionary<DateTime, List<Record>>>(); var testDict = new Dictionary<ulong, Dictionary<DateTime, List<Record>>>();
var options = new PartitionFileStorageOptions<PartitionKey, Record> var options = new PartitionFileSystemStorageOptions<PartitionKey, Record>
{ {
MaxDegreeOfParallelism = 1, MaxDegreeOfParallelism = 1,
DataConverter = new DataConverter(), DataConverter = new DataConverter(),
@ -86,7 +86,7 @@ namespace PartitionFileStorageTest
}; };
options.Partitions.Add(new Partition<PartitionKey>("data", p => p.Date.ToString("yyyyMMdd"))); options.Partitions.Add(new Partition<PartitionKey>("data", p => p.Date.ToString("yyyyMMdd")));
options.Partitions.Add(new Partition<PartitionKey>("ctn", p => p.Ctn.ToString().PadLeft(COUNT_NUMBERS, '0'))); options.Partitions.Add(new Partition<PartitionKey>("ctn", p => p.Ctn.ToString().PadLeft(COUNT_NUMBERS, '0')));
var storage = new PartitionFileStorage<PartitionKey, Record>(options); var storage = new PartitionFileSystemStorage<PartitionKey, Record>(options);
for (int i = 0; i < 50000; i++) for (int i = 0; i < 50000; i++)
{ {

@ -0,0 +1,11 @@
using System.Collections.Generic;
using System.IO;
namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{
public interface IPartitionDataConverter<TRecord>
{
IEnumerable<TRecord> ReadFromStorage(Stream stream);
void WriteToStorage(Stream stream, IEnumerable<TRecord> data);
}
}

@ -0,0 +1,12 @@
using System.Collections.Generic;
using System.Threading.Tasks;
namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{
public interface IPartitionFileStorage<TKey, TRecord>
{
Task WriteAsync(TKey key, IEnumerable<TRecord> records);
Task<IEnumerable<TRecord>> CollectAsync(IEnumerable<TKey> keys);
void Drop(TKey key);
}
}

@ -0,0 +1,18 @@
using System;
namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{
/// <summary>
/// Make part of full file path
/// </summary>
public class Partition<TKey>
{
public Partition(string name, Func<TKey, string> pathExtractor)
{
Name = name;
PathExtractor = pathExtractor;
}
public Func<TKey, string> PathExtractor { get; }
public string Name { get; }
}
}

@ -7,51 +7,14 @@ using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using ZeroLevel.Services.FileSystem; using ZeroLevel.Services.FileSystem;
namespace ZeroLevel.Services.Storages namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{ {
public class Partition<TKey> public class PartitionFileSystemStorage<TKey, TRecord>
{
public Partition(string name, Func<TKey, string> pathExtractor)
{
Name = name;
PathExtractor = pathExtractor;
}
public Func<TKey, string> PathExtractor { get; }
public string Name { get; }
}
/// <summary>
/// Write data to partitional data storage
/// </summary>
public interface IPartitionFileStorage<TKey, TRecord>
{
Task WriteAsync(TKey key, IEnumerable<TRecord> records);
Task<IEnumerable<TRecord>> CollectAsync(IEnumerable<TKey> keys);
void Drop(TKey key);
}
public interface IPartitionDataConverter<TRecord>
{
IEnumerable<TRecord> ConvertFromStorage(Stream stream);
void ConvertToStorage(Stream stream, IEnumerable<TRecord> data);
}
public class PartitionFileStorageOptions<TKey, TRecord>
{
public string RootFolder { get; set; }
public int MaxDegreeOfParallelism { get; set; } = Environment.ProcessorCount / 2;
public bool MergeFiles { get; set; } = false;
public int MergeFrequencyInMinutes { get; set; } = 180;
public bool UseCompression { get; set; } = false;
public IPartitionDataConverter<TRecord> DataConverter { get; set; }
public List<Partition<TKey>> Partitions { get; set; } = new List<Partition<TKey>>();
}
public class PartitionFileStorage<TKey, TRecord>
: IPartitionFileStorage<TKey, TRecord> : IPartitionFileStorage<TKey, TRecord>
{ {
private readonly PartitionFileStorageOptions<TKey, TRecord> _options; private readonly PartitionFileSystemStorageOptions<TKey, TRecord> _options;
public PartitionFileStorage(PartitionFileStorageOptions<TKey, TRecord> options) public PartitionFileSystemStorage(PartitionFileSystemStorageOptions<TKey, TRecord> options)
{ {
if (options.RootFolder == null) if (options.RootFolder == null)
throw new ArgumentNullException(nameof(options.RootFolder)); throw new ArgumentNullException(nameof(options.RootFolder));
@ -86,7 +49,7 @@ namespace ZeroLevel.Services.Storages
{ {
using (var stream = CreateReadStream(file)) using (var stream = CreateReadStream(file))
{ {
foreach (var item in _options.DataConverter.ConvertFromStorage(stream)) foreach (var item in _options.DataConverter.ReadFromStorage(stream))
{ {
set.Add(item); set.Add(item);
} }
@ -99,7 +62,7 @@ namespace ZeroLevel.Services.Storages
{ {
using (var stream = CreateWriteStream(key)) using (var stream = CreateWriteStream(key))
{ {
_options.DataConverter.ConvertToStorage(stream, records); _options.DataConverter.WriteToStorage(stream, records);
await stream.FlushAsync(); await stream.FlushAsync();
} }
} }
@ -125,7 +88,7 @@ namespace ZeroLevel.Services.Storages
var files = Directory.GetFiles(path); var files = Directory.GetFiles(path);
if (files != null && files.Length > 1) if (files != null && files.Length > 1)
{ {
// TODO
} }
} }
@ -171,7 +134,7 @@ namespace ZeroLevel.Services.Storages
{ {
var ms = new MemoryStream(); var ms = new MemoryStream();
using (var compressed = new GZipStream(stream, CompressionMode.Decompress, false)) using (var compressed = new GZipStream(stream, CompressionMode.Decompress, false))
{ {
compressed.CopyTo(ms); compressed.CopyTo(ms);
} }
ms.Position = 0; ms.Position = 0;

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{
public class PartitionFileSystemStorageOptions<TKey, TRecord>
{
public string RootFolder { get; set; }
public int MaxDegreeOfParallelism { get; set; } = Environment.ProcessorCount / 2;
public bool MergeFiles { get; set; } = false;
public int MergeFrequencyInMinutes { get; set; } = 180;
public bool UseCompression { get; set; } = false;
public IPartitionDataConverter<TRecord> DataConverter { get; set; }
public List<Partition<TKey>> Partitions { get; set; } = new List<Partition<TKey>>();
}
}

@ -6,17 +6,16 @@
</Description> </Description>
<Authors>ogoun</Authors> <Authors>ogoun</Authors>
<Company>ogoun</Company> <Company>ogoun</Company>
<AssemblyVersion>3.3.6.6</AssemblyVersion> <AssemblyVersion>3.3.6.7</AssemblyVersion>
<PackageReleaseNotes>Configuration binding refactoring. <PackageReleaseNotes>PartitionFileSystemStorage</PackageReleaseNotes>
Append custom parsing attribute.</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl> <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2022</Copyright> <Copyright>Copyright Ogoun 2022</Copyright>
<PackageLicenseUrl></PackageLicenseUrl> <PackageLicenseUrl></PackageLicenseUrl>
<PackageIconUrl></PackageIconUrl> <PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl> <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType> <RepositoryType>git</RepositoryType>
<Version>3.3.6.6</Version> <Version>3.3.6.7</Version>
<FileVersion>3.3.6.6</FileVersion> <FileVersion>3.3.6.7</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms> <Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon> <PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType> <DebugType>full</DebugType>

Loading…
Cancel
Save

Powered by TurnKey Linux.