using System.Diagnostics; using System.Text; using ZeroLevel.Services.PartitionStorage; namespace PartitionFileStorageTest { public class CallRecordParser { private static HashSet _partsOfNumbers = new HashSet { '*', '#', '+', '(', ')', '-' }; private StringBuilder sb = new StringBuilder(); private const string NO_VAL = null; private string ReadNumber(string line) { sb.Clear(); var started = false; foreach (var ch in line) { if (char.IsDigit(ch)) { if (started) { sb.Append(ch); } else if (ch != '0') { sb.Append(ch); started = true; } } else if (char.IsWhiteSpace(ch) || _partsOfNumbers.Contains(ch)) continue; else return NO_VAL; } if (sb.Length == 11 && sb[0] == '8') sb[0] = '7'; if (sb.Length == 3 || sb.Length == 4 || sb.Length > 10) return sb.ToString(); return NO_VAL; } private HashSet ReadNumbers(string line) { var result = new HashSet(); if (string.IsNullOrWhiteSpace(line) == false) { char STX = (char)0x0002; var values = line.Split(STX); if (values.Length > 0) { foreach (var val in values) { var number = ReadNumber(val); if (number != null) { result.Add(number); } } } } return result; } /// /// Парсинг строки исходного файла /// /// /// public CallRecord Parse(string line) { var parts = line.Split('\t'); if (parts.Length != 2) return null; var msisdn = ReadNumber(parts[0].Trim()); if (string.IsNullOrWhiteSpace(msisdn) == false) { var numbers = ReadNumbers(parts[1]); if (numbers != null && numbers.Count > 0) { return new CallRecord { Msisdn = msisdn, Msisdns = numbers }; } } return null; } } public class CallRecord { public string Msisdn; public HashSet Msisdns; } internal class Program { private class Metadata { public DateTime Date { get; set; } public bool Incoming { get; set; } } private static void BuildStore(string source, string root) { var options = new IStoreOptions { RootFolder = root, FilePartition = new StoreFilePartition("Last three digits", (ctn, date) => (ctn % 1000).ToString()), MergeFunction = list => { ulong s = 0; return Compressor.GetEncodedBytes(list.OrderBy(c => c), ref s); }, Partitions = new List> { new StoreCatalogPartition("Date", m => m.Date.ToString("yyyyMMdd")), new StoreCatalogPartition("Date", m => m.Incoming ? "incoming" : "outcoming") }, KeyComparer = (left, right) => left == right ? 0 : (left < right) ? -1 : 1, }; options.Index.Enabled = true; var store = new Store(options); var storeIncoming = store.CreateAccessor(new Metadata { Date = new DateTime(2022, 11, 08), Incoming = true }); var storeOutcoming = store.CreateAccessor(new Metadata { Date = new DateTime(2022, 11, 08), Incoming = false }); var parser = new CallRecordParser(); var sw = new Stopwatch(); sw.Start(); using (FileStream fs = File.Open(source, FileMode.Open, FileAccess.Read, FileShare.None)) { using (BufferedStream bs = new BufferedStream(fs, 1024 * 1024 * 64)) { using (StreamReader sr = new StreamReader(bs)) { string line; while ((line = sr.ReadLine()) != null) { var record = parser.Parse(line); if (record == null) continue; if (!string.IsNullOrWhiteSpace(record?.Msisdn ?? string.Empty) && ulong.TryParse(record.Msisdn, out var n)) { var ctns = record.Msisdns.ParseMsisdns().ToArray(); foreach (var ctn in ctns) { storeIncoming.Store(n, ctn); storeOutcoming.Store(ctn, n); } } } } } } sw.Stop(); Console.WriteLine($"Fill journal: {sw.ElapsedMilliseconds}ms"); sw.Restart(); storeIncoming.CompleteStoreAndRebuild(); storeOutcoming.CompleteStoreAndRebuild(); sw.Stop(); Console.WriteLine($"Rebuild journal to store: {sw.ElapsedMilliseconds}ms"); sw.Restart(); storeIncoming.RebuildIndex(); storeOutcoming.RebuildIndex(); sw.Stop(); Console.WriteLine($"Rebuild indexes: {sw.ElapsedMilliseconds}ms"); } private static void TestReading(string source, string root) { var options = new IStoreOptions { Index = new IndexOptions { Enabled = false, FileIndexCount = 64 }, RootFolder = root, FilePartition = new StoreFilePartition("Last three digits", (ctn, date) => (ctn % 1000).ToString()), MergeFunction = list => { ulong s = 0; return Compressor.GetEncodedBytes(list.OrderBy(c => c), ref s); }, Partitions = new List> { new StoreCatalogPartition("timestamp", m => m.Date.ToString("yyyyMMdd")), new StoreCatalogPartition("timestamp", m => m.Incoming ? "incoming" : "outcoming") }, KeyComparer = (left, right) => left == right ? 0 : (left < right) ? -1 : 1 }; var store = new Store(options); var request = new StoreSearchRequest { PartitionSearchRequests = new List> { new PartitionSearchRequest { Info = new Metadata { Date = new DateTime(2022, 11, 08), Incoming = true }, Keys = new ulong[] { 79645090604, 79645100604, 79643090604 } }, new PartitionSearchRequest { Info = new Metadata { Date = new DateTime(2022, 11, 08), Incoming = false }, Keys = new ulong[] { 79645090604, 79645100604, 79643090604 } } } }; var storeIncoming = store.CreateAccessor(new Metadata { Date = new DateTime(2022, 11, 08), Incoming = true }); Console.WriteLine($"Incoming data files: {storeIncoming.CountDataFiles()}"); var storeOutcoming = store.CreateAccessor(new Metadata { Date = new DateTime(2022, 11, 08), Incoming = false }); Console.WriteLine($"Outcoming data files: {storeOutcoming.CountDataFiles()}"); var sw = new Stopwatch(); sw.Start(); var result = store.Search(request).Result; foreach (var r in result.Results) { Console.WriteLine($"Incoming: {r.Key.Incoming}"); foreach (var mr in r.Value) { Console.WriteLine($"\tKey: {mr.Key}. Sucess: {mr.Found}"); if (mr.Found && mr.Value.Length > 0) { var ctns = Compressor.DecodeBytesContent(mr.Value); Console.WriteLine($"\t\t{string.Join(';', ctns)}"); } } } sw.Stop(); Console.WriteLine($"Search time: {sw.ElapsedMilliseconds}ms"); } private struct KeyIndex { public TKey Key { get; set; } public long Offset { get; set; } } static KeyIndex[] Generate(int count) { var arr = new KeyIndex[count]; for (int i = 0; i < count; i++) { arr[i] = new KeyIndex { Key = i * 3, Offset = i * 17 }; } return arr; } private static KeyIndex BinarySearchInIndex(KeyIndex[] index, long key, Func keyComparer, ref int position) { if (index == null || index.Length == 0) { return new KeyIndex { Key = key, Offset = 0 }; } int left = position; int right = index.Length - 1; int mid = 0; long test; while (left <= right) { mid = (int)Math.Floor((right + left) / 2d); test = index[mid].Key; var c = keyComparer(test, key); if (c < 0) { left = mid + 1; } else if (c > 0) { right = mid - 1; } else { position = mid; return index[mid]; } } position = mid; return index[mid]; } static void Main(string[] args) { var root = @"H:\temp"; var source = @"H:\319a9c31-d823-4dd1-89b0-7fb1bb9c4859.txt"; //BuildStore(source, root); TestReading(source, root); /* Func keyComparer = (left, right) => (left == right) ? 0 : (left < right) ? -1 : 1; var indexes = Generate(77); int position = 0; for (long i = 65; i < 700; i++) { var ind = BinarySearchInIndex(indexes, i, keyComparer, ref position); Console.WriteLine($"{i}: {ind.Offset}. [{ind.Key}]"); } */ Console.ReadKey(); } } }