using ZeroLevel; using ZeroLevel.Services.Serialization; using ZeroLevel.Services.Storages; using ZeroLevel.Services.Storages.PartitionFileSystemStorage; namespace PartitionFileStorageTest { internal class Program { public class PartitionKey { public DateTime Date { get; set; } public ulong Ctn { get; set; } } public class Record : IBinarySerializable { public string[] Hosts { get; set; } public void Deserialize(IBinaryReader reader) { this.Hosts = reader.ReadStringArray(); } public void Serialize(IBinaryWriter writer) { writer.WriteArray(this.Hosts); } } static Record GenerateRecord() { var record = new Record(); var rnd = new Random((int)Environment.TickCount); var count = rnd.Next(400); record.Hosts = new string[count]; for (int i = 0; i < count; i++) { record.Hosts[i] = Guid.NewGuid().ToString(); } return record; } static PartitionKey GenerateKey() { var key = new PartitionKey(); var rnd = new Random((int)Environment.TickCount); key.Ctn = (ulong)rnd.Next(1000); key.Date = DateTime.Now.AddDays(-rnd.Next(30)).Date; return key; } class DataConverter : IPartitionDataConverter { public IEnumerable ReadFromStorage(Stream stream) { var reader = new MemoryStreamReader(stream); while (reader.EOS == false) { yield return reader.Read(); } } public void WriteToStorage(Stream stream, IEnumerable data) { var writer = new MemoryStreamWriter(stream); foreach (var record in data) { writer.Write(record); } } } private static int COUNT_NUMBERS = ulong.MaxValue.ToString().Length; static void Main(string[] args) { var testDict = new Dictionary>>(); var options = new PartitionFileSystemStorageOptions { MaxDegreeOfParallelism = 1, DataConverter = new DataConverter(), UseCompression = true, MergeFiles = false, RootFolder = Path.Combine(Configuration.BaseDirectory, "root") }; options.Partitions.Add(new Partition("data", p => p.Date.ToString("yyyyMMdd"))); options.Partitions.Add(new Partition("ctn", p => p.Ctn.ToString().PadLeft(COUNT_NUMBERS, '0'))); var storage = new PartitionFileSystemStorage(options); for (int i = 0; i < 50000; i++) { if (i % 100 == 0) Console.WriteLine(i); var key = GenerateKey(); var record = GenerateRecord(); if (testDict.ContainsKey(key.Ctn) == false) { testDict[key.Ctn] = new Dictionary>(); } if (testDict[key.Ctn].ContainsKey(key.Date) == false) { testDict[key.Ctn][key.Date] = new List(); } testDict[key.Ctn][key.Date].Add(record); storage.WriteAsync(key, new[] { record }).Wait(); } foreach (var cpair in testDict) { foreach (var dpair in cpair.Value) { var key = new PartitionKey { Ctn = cpair.Key, Date = dpair.Key }; var data = storage.CollectAsync(new[] { key }).Result.ToArray(); var testData = dpair.Value; if (data.Length != testData.Count) { Console.WriteLine($"[{key.Date.ToString("yyyyMMdd")}] {key.Ctn} Wrong count. Expected: {testData.Count}. Got: {data.Length}"); } else { var datahosts = data.SelectMany(r => r.Hosts).OrderBy(s => s).ToArray(); var testhosts = testData.SelectMany(r => r.Hosts).OrderBy(s => s).ToArray(); if (datahosts.Length != testhosts.Length) { Console.WriteLine($"[{key.Date.ToString("yyyyMMdd")}] {key.Ctn}. Records not equals. Different hosts count"); } for (int i = 0; i < datahosts.Length; i++) { if (string.Compare(datahosts[i], testhosts[i], StringComparison.Ordinal) != 0) { Console.WriteLine($"[{key.Date.ToString("yyyyMMdd")}] {key.Ctn}. Records not equals. Different hosts"); } } } } } } } }