PartitionFileSystemStorage

pull/3/head
Ogoun 2 years ago
parent 829d39d854
commit a996f36abe

@ -76,7 +76,7 @@ namespace PartitionFileStorageTest
static void Main(string[] args)
{
var testDict = new Dictionary<ulong, Dictionary<DateTime, List<Record>>>();
var options = new PartitionFileStorageOptions<PartitionKey, Record>
var options = new PartitionFileSystemStorageOptions<PartitionKey, Record>
{
MaxDegreeOfParallelism = 1,
DataConverter = new DataConverter(),
@ -86,7 +86,7 @@ namespace PartitionFileStorageTest
};
options.Partitions.Add(new Partition<PartitionKey>("data", p => p.Date.ToString("yyyyMMdd")));
options.Partitions.Add(new Partition<PartitionKey>("ctn", p => p.Ctn.ToString().PadLeft(COUNT_NUMBERS, '0')));
var storage = new PartitionFileStorage<PartitionKey, Record>(options);
var storage = new PartitionFileSystemStorage<PartitionKey, Record>(options);
for (int i = 0; i < 50000; i++)
{

@ -0,0 +1,11 @@
using System.Collections.Generic;
using System.IO;
namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{
public interface IPartitionDataConverter<TRecord>
{
IEnumerable<TRecord> ReadFromStorage(Stream stream);
void WriteToStorage(Stream stream, IEnumerable<TRecord> data);
}
}

@ -0,0 +1,12 @@
using System.Collections.Generic;
using System.Threading.Tasks;
namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{
public interface IPartitionFileStorage<TKey, TRecord>
{
Task WriteAsync(TKey key, IEnumerable<TRecord> records);
Task<IEnumerable<TRecord>> CollectAsync(IEnumerable<TKey> keys);
void Drop(TKey key);
}
}

@ -0,0 +1,18 @@
using System;
namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{
/// <summary>
/// Make part of full file path
/// </summary>
public class Partition<TKey>
{
public Partition(string name, Func<TKey, string> pathExtractor)
{
Name = name;
PathExtractor = pathExtractor;
}
public Func<TKey, string> PathExtractor { get; }
public string Name { get; }
}
}

@ -7,51 +7,14 @@ using System.Linq;
using System.Threading.Tasks;
using ZeroLevel.Services.FileSystem;
namespace ZeroLevel.Services.Storages
namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{
public class Partition<TKey>
{
public Partition(string name, Func<TKey, string> pathExtractor)
{
Name = name;
PathExtractor = pathExtractor;
}
public Func<TKey, string> PathExtractor { get; }
public string Name { get; }
}
/// <summary>
/// Write data to partitional data storage
/// </summary>
public interface IPartitionFileStorage<TKey, TRecord>
{
Task WriteAsync(TKey key, IEnumerable<TRecord> records);
Task<IEnumerable<TRecord>> CollectAsync(IEnumerable<TKey> keys);
void Drop(TKey key);
}
public interface IPartitionDataConverter<TRecord>
{
IEnumerable<TRecord> ConvertFromStorage(Stream stream);
void ConvertToStorage(Stream stream, IEnumerable<TRecord> data);
}
public class PartitionFileStorageOptions<TKey, TRecord>
{
public string RootFolder { get; set; }
public int MaxDegreeOfParallelism { get; set; } = Environment.ProcessorCount / 2;
public bool MergeFiles { get; set; } = false;
public int MergeFrequencyInMinutes { get; set; } = 180;
public bool UseCompression { get; set; } = false;
public IPartitionDataConverter<TRecord> DataConverter { get; set; }
public List<Partition<TKey>> Partitions { get; set; } = new List<Partition<TKey>>();
}
public class PartitionFileStorage<TKey, TRecord>
public class PartitionFileSystemStorage<TKey, TRecord>
: IPartitionFileStorage<TKey, TRecord>
{
private readonly PartitionFileStorageOptions<TKey, TRecord> _options;
public PartitionFileStorage(PartitionFileStorageOptions<TKey, TRecord> options)
private readonly PartitionFileSystemStorageOptions<TKey, TRecord> _options;
public PartitionFileSystemStorage(PartitionFileSystemStorageOptions<TKey, TRecord> options)
{
if (options.RootFolder == null)
throw new ArgumentNullException(nameof(options.RootFolder));
@ -86,7 +49,7 @@ namespace ZeroLevel.Services.Storages
{
using (var stream = CreateReadStream(file))
{
foreach (var item in _options.DataConverter.ConvertFromStorage(stream))
foreach (var item in _options.DataConverter.ReadFromStorage(stream))
{
set.Add(item);
}
@ -99,7 +62,7 @@ namespace ZeroLevel.Services.Storages
{
using (var stream = CreateWriteStream(key))
{
_options.DataConverter.ConvertToStorage(stream, records);
_options.DataConverter.WriteToStorage(stream, records);
await stream.FlushAsync();
}
}
@ -125,7 +88,7 @@ namespace ZeroLevel.Services.Storages
var files = Directory.GetFiles(path);
if (files != null && files.Length > 1)
{
// TODO
}
}
@ -171,7 +134,7 @@ namespace ZeroLevel.Services.Storages
{
var ms = new MemoryStream();
using (var compressed = new GZipStream(stream, CompressionMode.Decompress, false))
{
{
compressed.CopyTo(ms);
}
ms.Position = 0;

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
namespace ZeroLevel.Services.Storages.PartitionFileSystemStorage
{
public class PartitionFileSystemStorageOptions<TKey, TRecord>
{
public string RootFolder { get; set; }
public int MaxDegreeOfParallelism { get; set; } = Environment.ProcessorCount / 2;
public bool MergeFiles { get; set; } = false;
public int MergeFrequencyInMinutes { get; set; } = 180;
public bool UseCompression { get; set; } = false;
public IPartitionDataConverter<TRecord> DataConverter { get; set; }
public List<Partition<TKey>> Partitions { get; set; } = new List<Partition<TKey>>();
}
}

@ -6,17 +6,16 @@
</Description>
<Authors>ogoun</Authors>
<Company>ogoun</Company>
<AssemblyVersion>3.3.6.6</AssemblyVersion>
<PackageReleaseNotes>Configuration binding refactoring.
Append custom parsing attribute.</PackageReleaseNotes>
<AssemblyVersion>3.3.6.7</AssemblyVersion>
<PackageReleaseNotes>PartitionFileSystemStorage</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2022</Copyright>
<PackageLicenseUrl></PackageLicenseUrl>
<PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<Version>3.3.6.6</Version>
<FileVersion>3.3.6.6</FileVersion>
<Version>3.3.6.7</Version>
<FileVersion>3.3.6.7</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType>

Loading…
Cancel
Save

Powered by TurnKey Linux.