diff --git a/ZeroLevel/Services/FileSystem/FSUtils.cs b/ZeroLevel/Services/FileSystem/FSUtils.cs index fe0df44..4e8dcbb 100644 --- a/ZeroLevel/Services/FileSystem/FSUtils.cs +++ b/ZeroLevel/Services/FileSystem/FSUtils.cs @@ -321,7 +321,7 @@ namespace ZeroLevel.Services.FileSystem public static void PackFolder(string sourceFolder, string zipPath, Func selector = null) { - var tmp = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + var tmp = FSUtils.GetAppLocalTemporaryDirectory(); var tmpDir = Directory.CreateDirectory(tmp); var files = new DirectoryInfo(sourceFolder) .GetFiles("*.*", SearchOption.AllDirectories) diff --git a/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs new file mode 100644 index 0000000..792f5e2 --- /dev/null +++ b/ZeroLevel/Services/PartitionStorage/Partition/CompactKeyStorePartitionBuilder.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using ZeroLevel.Services.FileSystem; +using ZeroLevel.Services.PartitionStorage.Interfaces; +using ZeroLevel.Services.Serialization; + +namespace ZeroLevel.Services.PartitionStorage.Partition +{ + internal sealed class CompactKeyStorePartitionBuilder + : BasePartition, IStorePartitionBuilder + { + private readonly Action _storeMethod; + + private long _totalRecords = 0; + + public long TotalRecords { get { return _totalRecords; } } + + public CompactKeyStorePartitionBuilder(StoreOptions options, + TMeta info, + IStoreSerializer serializer) + : base(options, info, serializer) + { + if (options == null) throw new ArgumentNullException(nameof(options)); + if (options.ThreadSafeWriting) + { + _storeMethod = StoreDirectSafe; + } + else + { + _storeMethod = StoreDirect; + } + } + + #region IStorePartitionBuilder + + + public void Store(TKey key, TInput value) + { + _storeMethod.Invoke(key, value); + Interlocked.Increment(ref _totalRecords); + } + + public void CompleteAdding() + { + CloseWriteStreams(); + } + + public void Compress() + { + var files = Directory.GetFiles(_catalog); + if (files != null && files.Length > 0) + { + Parallel.ForEach(files, file => CompressFile(file)); + } + } + public IEnumerable> Iterate() + { + var files = Directory.GetFiles(_catalog); + if (files != null && files.Length > 0) + { + foreach (var file in files) + { + if (TryGetReadStream(file, out var reader)) + { + using (reader) + { + while (reader.EOS == false) + { + var key = Serializer.KeyDeserializer.Invoke(reader); + var val = Serializer.InputDeserializer.Invoke(reader); + yield return new StorePartitionKeyValueSearchResult { Key = key, Value = val, Status = SearchResult.Success }; + } + } + } + } + } + } + public void RebuildIndex() => RebuildIndexes(); + #endregion + + #region Private methods + private void StoreDirect(TKey key, TInput value) + { + var groupKey = _options.GetFileName(key, _info); + if (TryGetWriteStream(groupKey, out var stream)) + { + Serializer.KeySerializer.Invoke(stream, key); + Thread.MemoryBarrier(); + Serializer.InputSerializer.Invoke(stream, value); + } + } + private void StoreDirectSafe(TKey key, TInput value) + { + var groupKey = _options.GetFileName(key, _info); + bool lockTaken = false; + if (TryGetWriteStream(groupKey, out var stream)) + { + Monitor.Enter(stream, ref lockTaken); + try + { + Serializer.KeySerializer.Invoke(stream, key); + Thread.MemoryBarrier(); + Serializer.InputSerializer.Invoke(stream, value); + } + finally + { + if (lockTaken) + { + Monitor.Exit(stream); + } + } + } + } + + internal void CompressFile(string file) + { + var dict = new Dictionary>(); + using (var reader = new MemoryStreamReader(new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.None, 4096 * 1024))) + { + while (reader.EOS == false) + { + var key = Serializer.KeyDeserializer.Invoke(reader); + if (false == dict.ContainsKey(key)) + { + dict[key] = new HashSet(); + } + if (reader.EOS) + { + break; + } + var input = Serializer.InputDeserializer.Invoke(reader); + dict[key].Add(input); + } + } + var tempFile = FSUtils.GetAppLocalTemporaryFile(); + using (var writer = new MemoryStreamWriter(new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096 * 1024))) + { + // sort for search acceleration + foreach (var pair in dict.OrderBy(p => p.Key)) + { + var v = _options.MergeFunction(pair.Value); + writer.SerializeCompatible(pair.Key); + writer.SerializeCompatible(v); + } + } + File.Delete(file); + File.Move(tempFile, file, true); + } + #endregion + } +} diff --git a/ZeroLevel/Services/PartitionStorage/Partition/InternalDirectHybridPartition.cs b/ZeroLevel/Services/PartitionStorage/Partition/InternalDirectHybridPartition.cs new file mode 100644 index 0000000..57f4532 --- /dev/null +++ b/ZeroLevel/Services/PartitionStorage/Partition/InternalDirectHybridPartition.cs @@ -0,0 +1,7 @@ +namespace ZeroLevel.Services.PartitionStorage.Partition +{ + internal class InternalDirectHybridPartition + { + + } +} diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs index 863cf1a..7da8789 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionAccessor.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using ZeroLevel.Services.FileSystem; using ZeroLevel.Services.PartitionStorage.Interfaces; using ZeroLevel.Services.PartitionStorage.Partition; @@ -324,8 +325,7 @@ namespace ZeroLevel.Services.PartitionStorage } // 2. Temporary file from ranges - var tempPath = Path.GetTempPath(); - var tempFile = Path.Combine(tempPath, Path.GetTempFileName()); + var tempFile = FSUtils.GetAppLocalTemporaryFile(); using (var readStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4096 * 1024)) { diff --git a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs index 95c89da..e3c655d 100644 --- a/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs +++ b/ZeroLevel/Services/PartitionStorage/Partition/StorePartitionBuilder.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using ZeroLevel.Services.FileSystem; using ZeroLevel.Services.PartitionStorage.Interfaces; using ZeroLevel.Services.PartitionStorage.Partition; using ZeroLevel.Services.Serialization; @@ -136,8 +137,7 @@ namespace ZeroLevel.Services.PartitionStorage dict[key].Add(input); } } - var tempPath = Path.GetTempPath(); - var tempFile = Path.Combine(tempPath, Path.GetTempFileName()); + var tempFile = FSUtils.GetAppLocalTemporaryFile(); using (var writer = new MemoryStreamWriter(new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096 * 1024))) { // sort for search acceleration diff --git a/ZeroLevel/ZeroLevel.csproj b/ZeroLevel/ZeroLevel.csproj index 886bf8e..4f63ace 100644 --- a/ZeroLevel/ZeroLevel.csproj +++ b/ZeroLevel/ZeroLevel.csproj @@ -6,16 +6,16 @@ ogoun ogoun - 3.3.8.4 - PartitionStorage. TotalRecords, count of recorded and merged values + 3.3.8.5 + Fix temporary file path generation https://github.com/ogoun/Zero/wiki Copyright Ogoun 2022 https://github.com/ogoun/Zero git - 3.3.8.4 - 3.3.8.4 + 3.3.8.5 + 3.3.8.5 AnyCPU;x64;x86 zero.png full