1. Configuration
- Fix ini reader
- New methods for merge configs
- New method GetOrCreateSection in ConfigurationSet
- New reader for read config from Environment Variables
- Refactoring
2. Partition storage
- Refactoring
pull/4/head
Ogoun 2 years ago
parent 06dc99456f
commit 99f11f87b6

@ -17,7 +17,7 @@ namespace PartitionFileStorageTest
internal class Program
{
// const int PAIRS_COUNT = 200_000_000;
const long PAIRS_COUNT = 100_000_000;
const long PAIRS_COUNT = 10_000_000;
private class Metadata
{
@ -298,70 +298,72 @@ namespace PartitionFileStorageTest
ulong totalData = 0;
ulong totalKeys = 0;
var readPart = store.CreateAccessor(meta);
/*
Log.Info("Test #1 reading");
foreach (var key in testKeys1)
using (var readPart = store.CreateAccessor(meta))
{
var result = readPart.Find(key);
totalData += (ulong)(result.Value?.Length ?? 0);
totalKeys++;
}
Log.Info($"\t\tFound: {totalKeys} keys. {totalData} bytes");
totalData = 0;
totalKeys = 0;
Log.Info("Test #1 remove by keys");
foreach (var key in testKeys1)
{
readPart.RemoveKey(key, false);
}
sw.Restart();
readPart.RebuildIndex();
sw.Stop();
Log.Info($"Rebuild indexes after remove: {sw.ElapsedMilliseconds}ms");
Log.Info("Test #1 reading after remove");
foreach (var key in testKeys1)
{
var result = readPart.Find(key);
totalData += (ulong)(result.Value?.Length ?? 0);
totalKeys++;
}
Log.Info($"\t\tFound: {totalKeys} keys. {totalData} bytes");
totalData = 0;
totalKeys = 0;
*/
Log.Info("Test #2 reading");
foreach (var key in testKeys2)
{
var result = readPart.Find(key);
totalData += (ulong)(result.Value?.Length ?? 0);
totalKeys++;
}
Log.Info($"\t\tFound: {totalKeys}/{Keys.Count} keys. {totalData} bytes");
totalData = 0;
totalKeys = 0;
Log.Info("Test #2 remove keys batch");
readPart.RemoveKeys(testKeys2);
foreach (var k in testKeys2)
{
Keys.TryRemove(k);
}
Log.Info("Test #2 reading after remove");
foreach (var key in testKeys2)
{
var result = readPart.Find(key);
totalData += (ulong)(result.Value?.Length ?? 0);
totalKeys++;
}
Log.Info($"\t\tFound: {totalKeys} keys. {totalData} bytes");
totalData = 0;
totalKeys = 0;
/*
Log.Info("Test #1 reading");
foreach (var key in testKeys1)
{
var result = readPart.Find(key);
totalData += (ulong)(result.Value?.Length ?? 0);
totalKeys++;
}
Log.Info($"\t\tFound: {totalKeys} keys. {totalData} bytes");
totalData = 0;
totalKeys = 0;
Log.Info("Test #1 remove by keys");
foreach (var key in testKeys1)
{
readPart.RemoveKey(key, false);
}
sw.Restart();
readPart.RebuildIndex();
sw.Stop();
Log.Info($"Rebuild indexes after remove: {sw.ElapsedMilliseconds}ms");
Log.Info("Test #1 reading after remove");
foreach (var key in testKeys1)
{
var result = readPart.Find(key);
totalData += (ulong)(result.Value?.Length ?? 0);
totalKeys++;
}
Log.Info($"\t\tFound: {totalKeys} keys. {totalData} bytes");
totalData = 0;
totalKeys = 0;
*/
Log.Info("Test #2 reading");
foreach (var key in testKeys2)
{
var result = readPart.Find(key);
totalData += (ulong)(result.Value?.Length ?? 0);
totalKeys++;
}
Log.Info($"\t\tFound: {totalKeys}/{Keys.Count} keys. {totalData} bytes");
totalData = 0;
totalKeys = 0;
Log.Info("Test #2 remove keys batch");
readPart.RemoveKeys(testKeys2);
foreach (var k in testKeys2)
{
Keys.TryRemove(k);
}
Log.Info("Test #2 reading after remove");
foreach (var key in testKeys2)
{
var result = readPart.Find(key);
totalData += (ulong)(result.Value?.Length ?? 0);
totalKeys++;
}
Log.Info($"\t\tFound: {totalKeys} keys. {totalData} bytes");
totalData = 0;
totalKeys = 0;
Log.Info("Iterator test");
foreach (var e in readPart.Iterate())
{
totalData += (ulong)(e.Value?.Length ?? 0);
totalKeys++;
Log.Info("Iterator test");
foreach (var e in readPart.Iterate())
{
totalData += (ulong)(e.Value?.Length ?? 0);
totalKeys++;
}
}
Log.Info($"\t\tFound: {totalKeys}/{Keys.Count} keys. {totalData} bytes");
store.Dispose();
@ -487,7 +489,7 @@ namespace PartitionFileStorageTest
static void Main(string[] args)
{
FaultCompressionTest(@"F:\Desktop\DATKA\DNS", new StoreMetadata { Date = new DateTime(2023, 01, 20) });
//FaultCompressionTest(@"F:\Desktop\DATKA\DNS", new StoreMetadata { Date = new DateTime(2023, 01, 20) });
var root = @"H:\temp";
var options = new StoreOptions<ulong, ulong, byte[], Metadata>

@ -5,7 +5,7 @@
<Platforms>AnyCPU;x64</Platforms>
<PlatformTarget>x64</PlatformTarget>
<DebugType>full</DebugType>
<Version>1.0.0.4</Version>
<Version>1.0.0.5</Version>
<Company>ogoun</Company>
<Authors>Ogoun</Authors>
<Copyright>Copyright Ogoun 2022</Copyright>

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using Xunit;
namespace ZeroLevel.UnitTests
@ -49,7 +50,7 @@ namespace ZeroLevel.UnitTests
// Assert
Assert.Equal("https://habr.ru", config.Url);
Assert.Equal(1000, config.BatchSize);
Assert.Contains(config.Sheme, t=>t.Equals("socks"));
Assert.Contains(config.Sheme, t => t.Equals("socks"));
Assert.Contains(config.Sheme, t => t.Equals("http"));
Assert.Contains(config.Sheme, t => t.Equals("https"));
@ -73,7 +74,7 @@ namespace ZeroLevel.UnitTests
Assert.Equal("System", config.Service.ServiceGroup);
Assert.Equal("service", config.Service.ServiceType);
}
[Fact]
public void NumbersTest()
{
@ -95,5 +96,52 @@ namespace ZeroLevel.UnitTests
Assert.Equal(f, set.Default.First<float>("f"));
Assert.Equal(i, set.Default.First<int>("i"));
}
[Fact]
public void MergeTest()
{
// ARRANGE
var set1 = Configuration.CreateSet();
set1.Default.Append("dk1", "1");
set1.Default.Append("dk1", "1");
set1["A"].Append("Ak1", "ak1");
set1["A"].Append("Ak2", "ak2");
var set2 = Configuration.CreateSet();
set2.Default.Append("dk1", "2");
set2["A"].Append("Ak1", "ak1");
var set3 = Configuration.CreateSet();
set3.Default.Append("dk1", "3");
set3["A"].Append("Ak1", "ak2");
// ACT 1
var mergedSet1 = Configuration.Merge(Services.Config.ConfigurationRecordExistBehavior.Append, set1, set2, set3);
// ASSERT 1
Assert.Equal(mergedSet1.Default["dk1"].Count(), 4);
Assert.Contains(mergedSet1.Default["dk1"], i => i == "1");
Assert.Contains(mergedSet1.Default["dk1"], i => i == "2");
Assert.Contains(mergedSet1.Default["dk1"], i => i == "3");
Assert.Equal(mergedSet1["A"]["Ak1"].Count(), 3);
Assert.Equal(mergedSet1["A"]["Ak2"].Count(), 1);
// ACT 2
var mergedSet2 = Configuration.Merge(Services.Config.ConfigurationRecordExistBehavior.IgnoreNew, set1, set2, set3);
// ASSERT 2
Assert.Equal(mergedSet2.Default["dk1"].Count(), 2);
Assert.Contains(mergedSet2.Default["dk1"], i => i == "1");
Assert.Equal(mergedSet2["A"]["Ak1"].Count(), 1);
Assert.Equal(mergedSet2["A"]["Ak2"].Count(), 1);
// ACT 3
var mergedSet3 = Configuration.Merge(Services.Config.ConfigurationRecordExistBehavior.Overwrite, set1, set2, set3);
// ASSERT 3
Assert.Equal(mergedSet3.Default["dk1"].Count(), 1);
Assert.Contains(mergedSet3.Default["dk1"], i => i == "3");
Assert.Equal(mergedSet3["A"]["Ak1"].Count(), 1);
Assert.Equal(mergedSet3["A"]["Ak2"].Count(), 1);
}
}
}

@ -6,30 +6,12 @@ using System.Linq;
using System.Net;
using System.Reflection;
using ZeroLevel.Services.Collections;
using ZeroLevel.Services.Invokation;
using ZeroLevel.Services.ObjectMapping;
using ZeroLevel.Services.Reflection;
using ZeroLevel.Services.Serialization;
namespace ZeroLevel.Services.Config
{
public interface IConfigRecordParser
{
object Parse(string line);
}
public class ConfigRecordParseAttribute
: Attribute
{
public IConfigRecordParser Parser { get; set; }
public ConfigRecordParseAttribute(Type parserType)
{
if (parserType == null) throw new ArgumentNullException(nameof(parserType));
Parser = (IConfigRecordParser)Activator.CreateInstance(parserType);
}
}
/// <summary>
/// Base configuration
/// </summary>
@ -489,6 +471,32 @@ namespace ZeroLevel.Services.Config
}
}
public void MergeFrom(IConfiguration config, ConfigurationRecordExistBehavior existRecordBehavior)
{
foreach (var key in config.Keys)
{
if (this.Contains(key))
{
switch (existRecordBehavior)
{
case ConfigurationRecordExistBehavior.Append:
this.Append(key, config[key]);
break;
case ConfigurationRecordExistBehavior.IgnoreNew:
continue;
case ConfigurationRecordExistBehavior.Overwrite:
this.Remove(key);
this.Append(key, config[key]);
break;
}
}
else
{
this.Append(key, config[key]);
}
}
}
public T Bind<T>() => (T)Bind(typeof(T));
public object Bind(Type type)
{

@ -3,7 +3,6 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using ZeroLevel.Services.ObjectMapping;
using ZeroLevel.Services.Reflection;
using ZeroLevel.Services.Serialization;
namespace ZeroLevel.Services.Config
@ -122,6 +121,17 @@ namespace ZeroLevel.Services.Config
throw new KeyNotFoundException("Section not found: " + sectionName);
}
public IConfiguration GetOrCreateSection(string sectionName)
{
var key = GetKey(sectionName);
IConfiguration exists;
if (_sections.TryGetValue(key, out exists))
{
return exists;
}
_sections.TryAdd(key, new BaseConfiguration());
return _sections[key];
}
public bool ContainsSection(string sectionName)
{
var key = GetKey(sectionName);

@ -0,0 +1,16 @@
using System;
namespace ZeroLevel.Services.Config
{
public class ConfigRecordParseAttribute
: Attribute
{
public IConfigRecordParser Parser { get; set; }
public ConfigRecordParseAttribute(Type parserType)
{
if (parserType == null) throw new ArgumentNullException(nameof(parserType));
Parser = (IConfigRecordParser)Activator.CreateInstance(parserType);
}
}
}

@ -131,11 +131,24 @@ namespace ZeroLevel
#endregion Factory
#region Read configuration
public static IConfiguration ReadFromEnvironmentVariables()
{
try
{
return new EnvironmentVariablesConfigReader().ReadConfiguration();
}
catch (Exception ex)
{
Log.Error(ex, $"[Configuration.ReadFromEnvironmentVariables] Can't read environment variables");
throw;
}
}
/// <summary>
/// Creating a configuration from the AppSettings section of the app.config or web.config file
/// </summary>
/// <returns>Configuration</returns>
public static IConfiguration ReadFromApplicationConfig()
public static IConfiguration ReadFromApplicationConfig()
{
try
{
@ -164,8 +177,8 @@ namespace ZeroLevel
/// Creating a configuration from the AppSettings section of the app.config file or web.config, is supplemented by the 'ConnectionStrings' section
/// </summary>
/// <returns>Configuration</returns>
public static IConfigurationSet ReadSetFromApplicationConfig()
{
public static IConfigurationSet ReadSetFromApplicationConfig()
{
try
{
return new ApplicationConfigReader().ReadConfigurationSet();
@ -193,7 +206,7 @@ namespace ZeroLevel
/// Creating a configuration from the AppSettings section of the app.config or web.config file
/// </summary>
/// <returns>Configuration</returns>
public static IConfiguration ReadFromApplicationConfig(string configFilePath)
public static IConfiguration ReadFromApplicationConfig(string configFilePath)
{
try
{
@ -222,7 +235,7 @@ namespace ZeroLevel
/// Creating a configuration from the AppSettings section of the app.config file or web.config, is supplemented by the 'ConnectionStrings' section
/// </summary>
/// <returns>Configuration</returns>
public static IConfigurationSet ReadSetFromApplicationConfig(string configFilePath)
public static IConfigurationSet ReadSetFromApplicationConfig(string configFilePath)
{
try
{
@ -252,7 +265,7 @@ namespace ZeroLevel
/// </summary>
/// <param name="path">Path to the ini file</param>
/// <returns>Configuration</returns>
public static IConfiguration ReadFromIniFile(string path)
public static IConfiguration ReadFromIniFile(string path)
{
try
{
@ -282,7 +295,7 @@ namespace ZeroLevel
/// </summary>
/// <param name="path">Path to the ini file</param>
/// <returns>Configuration</returns>
public static IConfigurationSet ReadSetFromIniFile(string path)
public static IConfigurationSet ReadSetFromIniFile(string path)
{
try
{
@ -312,7 +325,7 @@ namespace ZeroLevel
/// </summary>
/// <param name="args">Command line parameters</param>
/// <returns>Configuration</returns>
public static IConfiguration ReadFromCommandLine(string[] args)
public static IConfiguration ReadFromCommandLine(string[] args)
{
try
{
@ -388,6 +401,32 @@ namespace ZeroLevel
}
#endregion Read configuration
public static IConfiguration Merge(ConfigurationRecordExistBehavior existRecordBehavior, params IConfiguration[] configurations)
{
var result = Configuration.Create();
foreach (var configuration in configurations)
{
result.MergeFrom(configuration, existRecordBehavior);
}
return result;
}
public static IConfigurationSet Merge(ConfigurationRecordExistBehavior existRecordBehavior, params IConfigurationSet[] configurationSets)
{
var result = Configuration.CreateSet();
foreach (var set in configurationSets)
{
foreach (var sectionName in set.SectionNames)
{
var section = result.GetOrCreateSection(sectionName);
section.MergeFrom(set[sectionName], existRecordBehavior);
}
}
return result;
}
#region Write configuration
/// <summary>

@ -0,0 +1,22 @@
namespace ZeroLevel.Services.Config
{
/// <summary>
/// Merge behavior when keys match
/// </summary>
public enum ConfigurationRecordExistBehavior
: int
{
/// <summary>
/// Add values to existing values
/// </summary>
Append = 0,
/// <summary>
/// Overwrite existing values
/// </summary>
Overwrite = 1,
/// <summary>
/// Ignore new values
/// </summary>
IgnoreNew = 2
}
}

@ -0,0 +1,7 @@
namespace ZeroLevel.Services.Config
{
public interface IConfigRecordParser
{
object Parse(string line);
}
}

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using ZeroLevel.Services.Config;
using ZeroLevel.Services.Serialization;
namespace ZeroLevel
@ -141,6 +142,7 @@ namespace ZeroLevel
#endregion Create, Clean, Delete
void CopyTo(IConfiguration config);
void MergeFrom(IConfiguration config, ConfigurationRecordExistBehavior existRecordBehavior);
T Bind<T>();
object Bind(Type type);

@ -57,6 +57,8 @@ namespace ZeroLevel
/// <returns>Data section</returns>
IConfiguration GetSection(string sectionName);
IConfiguration GetOrCreateSection(string sectionName);
/// <summary>
/// Check for a section by name
/// </summary>

@ -0,0 +1,26 @@
using System;
namespace ZeroLevel.Services.Config.Implementation
{
internal sealed class EnvironmentVariablesConfigReader
: IConfigurationReader
{
public IConfiguration ReadConfiguration()
{
var result = Configuration.Create();
var enumerator = Environment.GetEnvironmentVariables().GetEnumerator();
while (enumerator.MoveNext())
{
string key = (string)enumerator.Entry.Key;
string value = ((string?)enumerator.Entry.Value) ?? string.Empty;
result.Append(key, value);
}
return result;
}
public IConfigurationSet ReadConfigurationSet()
{
return Configuration.CreateSet(ReadConfiguration());
}
}
}

@ -53,8 +53,13 @@ namespace ZeroLevel.Services.Config.Implementation
key = originalKey.ToLower(CultureInfo.InvariantCulture);
value = string.Empty;
}
if (key[0].Equals(';') || key[0].Equals('#'))
if (key[0].Equals(';') || key[0].Equals('#') || key[0].Equals('/'))
continue;
// Remove quotes
if (value.Length > 1 && value[0] == '"' && value[value.Length - 1] == '"')
{
value = value.Substring(1, value.Length - 2);
}
if (string.IsNullOrEmpty(value) && key[0].Equals('[') && key[key.Length - 1].Equals(']'))
{
sectionName = originalKey.Trim('[', ']');
@ -100,8 +105,13 @@ namespace ZeroLevel.Services.Config.Implementation
key = originalKey.ToLower(CultureInfo.InvariantCulture);
value = string.Empty;
}
if (key[0].Equals(';') || key[0].Equals('#'))
if (key[0].Equals(';') || key[0].Equals('#') || key[0].Equals('/'))
continue;
// Remove quotes
if (value.Length > 1 && value[0] == '"' && value[value.Length - 1] == '"')
{
value = value.Substring(1, value.Length - 2);
}
if (string.IsNullOrEmpty(value) && key[0].Equals('[') && key[key.Length - 1].Equals(']'))
{
sectionName = originalKey.Trim('[', ']');

@ -111,7 +111,13 @@ namespace ZeroLevel.Services.PartitionStorage
{
if (_enableIndexInMemoryCachee)
{
_phisicalFileAccessorCachee.DropAllIndexReaders();
if (Directory.Exists(_indexFolder))
{
foreach (var file in Directory.GetFiles(_indexFolder))
{
_phisicalFileAccessorCachee.DropIndexReader(file);
}
}
}
}

@ -51,9 +51,12 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
public void Dispose()
{
CloseWriteStreams();
Release();
}
#endregion
public abstract void Release();
/// <summary>
/// Rebuild indexes for all files
/// </summary>

@ -169,6 +169,17 @@ namespace ZeroLevel.Services.PartitionStorage.Partition
PhisicalFileAccessorCachee.UnlockFile(file);
}
}
public override void Release()
{
if (Directory.Exists(_catalog))
{
foreach (var file in Directory.GetFiles(_catalog))
{
PhisicalFileAccessorCachee.DropDataReader(file);
}
}
}
#endregion
}
}

@ -477,5 +477,17 @@ namespace ZeroLevel.Services.PartitionStorage
target.Write(buffer, 0, buffer.Length);
}
#endregion
public override void Release()
{
if (Directory.Exists(_catalog))
{
foreach (var file in Directory.GetFiles(_catalog))
{
PhisicalFileAccessorCachee.DropDataReader(file);
}
}
Indexes.ResetCachee();
}
}
}

@ -5,7 +5,6 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using ZeroLevel.Services.FileSystem;
using ZeroLevel.Services.Memory;
using ZeroLevel.Services.PartitionStorage.Interfaces;
using ZeroLevel.Services.PartitionStorage.Partition;
using ZeroLevel.Services.Serialization;
@ -199,6 +198,10 @@ namespace ZeroLevel.Services.PartitionStorage
PhisicalFileAccessorCachee.UnlockFile(file);
}
}
public override void Release()
{
}
#endregion
}
}

@ -6,16 +6,16 @@
</Description>
<Authors>ogoun</Authors>
<Company>ogoun</Company>
<AssemblyVersion>3.3.9.3</AssemblyVersion>
<PackageReleaseNotes>Update Bootstrap</PackageReleaseNotes>
<AssemblyVersion>3.3.9.5</AssemblyVersion>
<PackageReleaseNotes>Partition storage. Fix.</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2023</Copyright>
<PackageLicenseUrl></PackageLicenseUrl>
<PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<Version>3.3.9.3</Version>
<FileVersion>3.3.9.3</FileVersion>
<Version>3.3.9.5</Version>
<FileVersion>3.3.9.5</FileVersion>
<Platforms>AnyCPU;x64;x86</Platforms>
<PackageIcon>zero.png</PackageIcon>
<DebugType>full</DebugType>

Loading…
Cancel
Save

Powered by TurnKey Linux.