// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. using Microsoft.Win32.SafeHandles; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; namespace FASTER.core { /// /// Sector aligned memory allocator /// public unsafe class SectorAlignedMemory { /// /// Actual buffer /// public byte[] buffer; /// /// Handle /// internal GCHandle handle; /// /// Offset /// public int offset; /// /// Aligned pointer /// public byte* aligned_pointer; /// /// Valid offset /// public int valid_offset; /// /// Required bytes /// public int required_bytes; /// /// Available bytes /// public int available_bytes; internal int level; internal SectorAlignedBufferPool pool; /// /// Return /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Return() { pool.Return(this); } /// /// Get valid pointer /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public byte* GetValidPointer() { return aligned_pointer + valid_offset; } /// /// ToString /// /// public override string ToString() { return string.Format("{0} {1} {2} {3} {4}", (long)aligned_pointer, offset, valid_offset, required_bytes, available_bytes); } } /// /// SectorAlignedBufferPool is a pool of memory. /// Internally, it is organized as an array of concurrent queues where each concurrent /// queue represents a memory of size in particular range. queue[i] contains memory /// segments each of size (2^i * sectorSize). /// public class SectorAlignedBufferPool { /// /// Disable buffer pool /// public static bool Disabled = false; private const int levels = 32; private readonly int recordSize; private readonly int sectorSize; private readonly ConcurrentQueue[] queue; /// /// Constructor /// /// Record size /// Sector size public SectorAlignedBufferPool(int recordSize, int sectorSize) { queue = new ConcurrentQueue[levels]; this.recordSize = recordSize; this.sectorSize = sectorSize; } /// /// Return /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Return(SectorAlignedMemory page) { Debug.Assert(queue[page.level] != null); page.available_bytes = 0; page.required_bytes = 0; page.valid_offset = 0; Array.Clear(page.buffer, 0, page.buffer.Length); if (!Disabled) queue[page.level].Enqueue(page); else { page.handle.Free(); page.buffer = null; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int Position(int v) { if (v == 1) return 0; v--; int r = 0; // r will be lg(v) while (true) // unroll for more speed... { v = v >> 1; if (v == 0) break; r++; } return r + 1; } /// /// Get buffer /// /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public unsafe SectorAlignedMemory Get(int numRecords) { int requiredSize = sectorSize + (((numRecords) * recordSize + (sectorSize - 1)) & ~(sectorSize - 1)); int index = Position(requiredSize / sectorSize); if (queue[index] == null) { var localPool = new ConcurrentQueue(); Interlocked.CompareExchange(ref queue[index], localPool, null); } if (!Disabled && queue[index].TryDequeue(out SectorAlignedMemory page)) { return page; } page = new SectorAlignedMemory { level = index, buffer = new byte[sectorSize * (1 << index)] }; page.handle = GCHandle.Alloc(page.buffer, GCHandleType.Pinned); page.aligned_pointer = (byte*)(((long)page.handle.AddrOfPinnedObject() + (sectorSize - 1)) & ~(sectorSize - 1)); page.offset = (int) ((long)page.aligned_pointer - (long)page.handle.AddrOfPinnedObject()); page.pool = this; return page; } /// /// Free buffer /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Free() { for (int i = 0; i < levels; i++) { if (queue[i] == null) continue; while (queue[i].TryDequeue(out SectorAlignedMemory result)) { result.handle.Free(); result.buffer = null; } } } /// /// Print pool contents /// public void Print() { for (int i = 0; i < levels; i++) { if (queue[i] == null) continue; foreach (var item in queue[i]) { Console.WriteLine(" " + item.ToString()); } } } } }