Added ZPriorityQueue

ZPriorityQueue and unittests

Tests/HNSWDemo/HNSWDemo.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="System.Drawing.Common" Version="7.0.0" />
+    <PackageReference Include="System.Drawing.Common" Version="8.0.1" />
   </ItemGroup>
 
   <ItemGroup>

Tests/PartitionFileStorageTest/PartitionFileStorageTest.csproj

@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.FASTER.Core" Version="2.6.0" />
+    <PackageReference Include="Microsoft.FASTER.Core" Version="2.6.1" />
   </ItemGroup>
 
   <ItemGroup>

Tests/Qdrant.Test/Qdrant.Test.csproj

@@ -8,8 +8,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Google.Protobuf" Version="3.24.3" />
+    <PackageReference Include="Google.Protobuf" Version="3.25.2" />
-    <PackageReference Include="Grpc.Net.Client" Version="2.57.0" />
+    <PackageReference Include="Grpc.Net.Client" Version="2.60.0" />
   </ItemGroup>
 
   <ItemGroup>

Tests/ZeroLevel.UnitTests/PriorityQueueTests.cs

@@ -0,0 +1,105 @@
+using Xunit;
+using ZeroLevel.Services.Collections;
+
+namespace ZeroLevel.UnitTests
+{
+    public class PriorityQueueTests
+    {
+        class TestItem
+        {
+            public string Value;
+        }
+
+        [Fact]
+        public void AllItemsCorrectHandleTest()
+        {
+            // Arrange
+            var q = new ZPriorityQueue<string>(s => new PriorityQueueObjectHandleResult { IsCompleted = true, CanBeSkipped = false });
+            q.Append("s0", 0);
+            q.Append("s1", 10);
+            q.Append("s2", 4);
+            q.Append("s3", 1);
+            q.Append("s4", 6);
+            q.Append("s5", 2);
+            q.Append("s6", 7);
+
+            // Assert
+            Assert.Equal("s0", q.HandleCurrentItem());
+            Assert.Equal("s3", q.HandleCurrentItem());
+            Assert.Equal("s5", q.HandleCurrentItem());
+            Assert.Equal("s2", q.HandleCurrentItem());
+            Assert.Equal("s4", q.HandleCurrentItem());
+            Assert.Equal("s6", q.HandleCurrentItem());
+            Assert.Equal("s1", q.HandleCurrentItem());
+        }
+
+        [Fact]
+        public void SkipItemsHandleTest()
+        {
+            bool skipped = false;
+            // Arrange
+            var q = new ZPriorityQueue<string>(s =>
+            {
+                if (s.Equals("s5") && !skipped)
+                { 
+                    skipped = true;
+                    return new PriorityQueueObjectHandleResult { IsCompleted = false, CanBeSkipped = true };
+                }
+                return new PriorityQueueObjectHandleResult { IsCompleted = true, CanBeSkipped = false };
+            });
+            q.Append("s0", 0);
+            q.Append("s1", 10);
+            q.Append("s2", 4);
+            q.Append("s3", 1);
+            q.Append("s4", 6);
+            q.Append("s5", 2);
+            q.Append("s6", 7);
+
+            // Assert
+            Assert.Equal("s0", q.HandleCurrentItem());
+            Assert.Equal("s3", q.HandleCurrentItem());            
+            
+            Assert.Equal("s2", q.HandleCurrentItem());
+            Assert.Equal("s5", q.HandleCurrentItem());
+
+            Assert.Equal("s4", q.HandleCurrentItem());
+            Assert.Equal("s6", q.HandleCurrentItem());
+            Assert.Equal("s1", q.HandleCurrentItem());
+        }
+
+        [Fact]
+        public void NoSkipItemsIncorrectHandleTest()
+        {
+            bool skipped = false;
+            // Arrange
+            var q = new ZPriorityQueue<string>(s =>
+            {
+                if (s.Equals("s5") && !skipped)
+                {
+                    skipped = true;
+                    return new PriorityQueueObjectHandleResult { IsCompleted = false, CanBeSkipped = false };
+                }
+                return new PriorityQueueObjectHandleResult { IsCompleted = true, CanBeSkipped = false };
+            });
+            q.Append("s0", 0);
+            q.Append("s1", 10);
+            q.Append("s2", 4);
+            q.Append("s3", 1);
+            q.Append("s4", 6);
+            q.Append("s5", 2);
+            q.Append("s6", 7);
+
+            // Assert
+            Assert.Equal("s0", q.HandleCurrentItem());
+            Assert.Equal("s3", q.HandleCurrentItem());
+
+            Assert.Equal(null, q.HandleCurrentItem());
+            Assert.Equal("s5", q.HandleCurrentItem());
+            Assert.Equal("s2", q.HandleCurrentItem());            
+
+            Assert.Equal("s4", q.HandleCurrentItem());
+            Assert.Equal("s6", q.HandleCurrentItem());
+            Assert.Equal("s1", q.HandleCurrentItem());
+        }
+    }
+}

Tests/ZeroLevel.UnitTests/ZeroLevel.UnitTests.csproj

@@ -9,9 +9,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-    <PackageReference Include="xunit" Version="2.5.1" />
+    <PackageReference Include="xunit" Version="2.6.6" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.1">
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.6">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

ZeroLevel.HNSW/PHNSW/HLevel.cs

@@ -0,0 +1,61 @@
+using System;
+
+namespace ZeroLevel.HNSW.PHNSW
+{
+    internal class HLevel<TPayload>
+        : IPHNSWLevel<TPayload>
+    {
+        private readonly float _distance;
+        public HLevel(float distance)
+        {
+            _distance = distance;
+        }
+
+        public Node<TPayload> Node { get; set; } = null;
+        public IPHNSWLevel<TPayload> NextLevelA { get; set; }
+        public IPHNSWLevel<TPayload> NextLevelB { get; set; }
+
+        private float _abDistance = float.MinValue;
+
+        public void Add(Node<TPayload> node)
+        {
+            if (NextLevelA.Node == null) { NextLevelA.Node = node; }
+            else if (NextLevelB.Node == null)
+            {
+                NextLevelB.Node = node;
+                _abDistance = PHNSWMetric.CosineDistance(NextLevelA.Node.Vector, NextLevelB.Node.Vector);
+            }
+            else
+            {
+                var an = PHNSWMetric.CosineDistance(NextLevelA.Node.Vector, node.Vector);
+                var bn = PHNSWMetric.CosineDistance(NextLevelB.Node.Vector, node.Vector);
+
+                var abDiff = Math.Abs(_distance - _abDistance);
+                var anDiff = Math.Abs(_distance - an);
+                var bnDiff = Math.Abs(_distance - bn);
+
+                if (abDiff < anDiff && abDiff < bnDiff)
+                {
+                    if (an < bn)
+                    {
+                        NextLevelA.Add(node);
+                    }
+                    else
+                    {
+                        NextLevelB.Add(node);
+                    }
+                }
+                else if (anDiff < bnDiff && anDiff < abDiff)
+                {
+                    NextLevelA.Node = node;
+                    NextLevelA.Add(node);
+                }
+                else
+                {
+                    NextLevelB.Node = node;
+                    NextLevelB.Add(node);
+                }
+            }
+        }
+    }
+}

ZeroLevel.HNSW/PHNSW/IPHNSWLevel.cs

@@ -0,0 +1,8 @@
+namespace ZeroLevel.HNSW.PHNSW
+{
+    public interface IPHNSWLevel<TPayload>
+    {
+        void Add(IPHNSWLevel<TPayload> prevLayer, Node<TPayload> node);
+        Node<TPayload> Node { get; internal set; }
+    }
+}

ZeroLevel.HNSW/PHNSW/Node.cs

@@ -0,0 +1,12 @@
+using ZeroLevel.DocumentObjectModel.Flow;
+
+namespace ZeroLevel.HNSW.PHNSW
+{
+    public class Node <TPayload>
+    {
+        public float[] Vector { get; set; }
+        public TPayload Payload { get; set; }
+
+        public List<Node<TPayload>> Neighbors { get; }
+    }
+}

ZeroLevel.HNSW/PHNSW/PHNSWBuilder.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ZeroLevel.HNSW.PHNSW
+{
+    public static class PHNSWBuilder
+    {
+        public static IPHNSWLevel<TPayload> Build<TPayload>(int levels)
+        {
+            var distance = 0.33f;
+            var root = new HLevel<TPayload>(distance);
+            var horizontalLayers = new List<HLevel<TPayload>>(new[] { root });
+            for (var i = 0; i < levels; i++)
+            {
+                distance /= 2.0f;
+                var nextList = new List<HLevel<TPayload>>();
+                foreach (var layer in horizontalLayers)
+                {
+                    var a = new HLevel<TPayload>(distance);
+                    var b = new HLevel<TPayload>(distance);
+                    layer.NextLevelA = a;
+                    layer.NextLevelB = b;
+                    nextList.Add(a); 
+                    nextList.Add(b);
+                }
+                horizontalLayers = nextList;
+            }
+            var uwLevel = new UWLevel<TPayload>();
+
+        }
+    }
+}

ZeroLevel.HNSW/PHNSW/PHNSWMetric.cs

@@ -0,0 +1,28 @@
+using System;
+
+namespace ZeroLevel.HNSW.PHNSW
+{
+    internal static class PHNSWMetric
+    {
+        internal static float CosineDistance(float[] u, float[] v)
+        {
+            if (u.Length != v.Length)
+            {
+                throw new ArgumentException("Vectors have non-matching dimensions");
+            }
+
+            float dot = 0.0f;
+            float nru = 0.0f;
+            float nrv = 0.0f;
+            for (int i = 0; i < u.Length; ++i)
+            {
+                dot += u[i] * v[i];
+                nru += u[i] * u[i];
+                nrv += v[i] * v[i];
+            }
+
+            var similarity = dot / (float)(Math.Sqrt(nru) * Math.Sqrt(nrv));
+            return 1 - similarity;
+        }
+    }
+}

ZeroLevel.HNSW/PHNSW/UWLevel.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ZeroLevel.HNSW.PHNSW
+{
+    public class UWLevel<TPayload>
+        :IPHNSWLevel<TPayload>
+    {
+    }
+}

ZeroLevel.NN/ZeroLevel.NN.csproj

@@ -35,10 +35,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Aurigma.GraphicsMill.Core.x64" Version="11.4.3" />
+    <PackageReference Include="Aurigma.GraphicsMill.Core.x64" Version="11.5.23" />
-    <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.15.1" />
+    <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.17.0" />
-    <PackageReference Include="SixLabors.ImageSharp" Version="3.0.2" />
+    <PackageReference Include="SixLabors.ImageSharp" Version="3.1.2" />
-    <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="2.0.0" />
+    <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="2.1.1" />
   </ItemGroup>
 
   <ItemGroup>

ZeroLevel.Qdrant.GrpcClient/ZeroLevel.Qdrant.GrpcClient.csproj

@@ -11,9 +11,9 @@
   </PropertyGroup>
 
 	<ItemGroup>
-		<PackageReference Include="Google.Protobuf" Version="3.24.3" />
+		<PackageReference Include="Google.Protobuf" Version="3.25.2" />
-		<PackageReference Include="Grpc.Net.Client" Version="2.57.0" />
+		<PackageReference Include="Grpc.Net.Client" Version="2.60.0" />
-		<PackageReference Include="Grpc.Tools" Version="2.58.0">
+		<PackageReference Include="Grpc.Tools" Version="2.61.0">
 		  <PrivateAssets>all</PrivateAssets>
 		  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
 		</PackageReference>

ZeroLevel.SQL/ZeroLevel.SQL.csproj

@@ -26,8 +26,8 @@
 
   <ItemGroup>
     <PackageReference Include="System.Data.Common" Version="4.3.0" />
-    <PackageReference Include="System.Data.SqlClient" Version="4.8.5" />
+    <PackageReference Include="System.Data.SqlClient" Version="4.8.6" />
-    <PackageReference Include="System.Security.Permissions" Version="7.0.0" />
+    <PackageReference Include="System.Security.Permissions" Version="8.0.0" />
   </ItemGroup>
 
   <ItemGroup>

ZeroLevel/DataStructures/BitMapCardTable.cs

@@ -0,0 +1,78 @@
+using System;
+using System.Collections.Generic;
+
+namespace ZeroLevel.DataStructures
+{
+    public sealed class BitMapCardTable
+    {
+        private readonly long[] _bitmap;
+        public BitMapCardTable(int N)
+        {
+            var count = N / 64 + (N % 64 == 0 ? 0 : 1);
+            _bitmap = new long[count];
+        }
+
+        public bool this[int index]
+        {
+            get
+            {
+                if (index < 0) throw new IndexOutOfRangeException();
+                var i = index / 64;
+                var n = _bitmap[i];
+                var bit_index = index % 64;
+                return (n & (1L << bit_index)) != 0;
+            }
+            set
+            {
+                if (index < 0) throw new IndexOutOfRangeException();
+                var i = index / 64;
+                var bit_index = index % 64;
+                if (value)
+                {
+                    _bitmap[i] = _bitmap[i] | (1L << bit_index);
+                }
+                else
+                {
+                    _bitmap[i] = _bitmap[i] & ~(1L << bit_index);
+                }
+            }
+        }
+
+        public void ResetTo(bool value)
+        {
+            if (value)
+            {
+                for (int i = 0; i < _bitmap.Length; i++)
+                {
+                    _bitmap[i] = long.MaxValue;
+                    _bitmap[i] |= 1L << 63;
+                }
+            }
+            else
+            {
+                for (int i = 0; i < _bitmap.Length; i++)
+                {
+                    _bitmap[i] = 0;
+                }
+            }
+        }
+
+        public IEnumerable<int> GetSetIndexes()
+        {
+            for (int i = 0; i < _bitmap.Length; i++)
+            {
+                if (_bitmap[i] != 0)
+                {
+                    var start = i * 64;
+                    for (int offset = 0; offset < 64; offset++)
+                    {
+                        if ((_bitmap[i] & (1L << offset)) != 0)
+                        {
+                            yield return start + offset;
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

ZeroLevel/Services/Collections/ZPriorityQueue.cs

@@ -0,0 +1,127 @@
+using System;
+
+namespace ZeroLevel.Services.Collections
+{
+    public interface IPriorityQueue<T>
+    {
+        int Count { get; }
+        void Append(T item, int priority);
+        T HandleCurrentItem();
+    }
+
+    public struct PriorityQueueObjectHandleResult
+    {
+        public bool IsCompleted;
+        public bool CanBeSkipped;
+    }
+
+    public class ZPriorityQueue<T>
+        : IPriorityQueue<T>
+    {
+        private sealed class PriorityQueueObject<T>
+        {
+            public readonly int Priority;
+
+            public readonly T Value;
+
+            public PriorityQueueObject<T> Next;
+
+            public PriorityQueueObject(T val, int priority)
+            {
+                Value = val;
+                Priority = priority;
+            }
+        }
+
+        private readonly Func<T, PriorityQueueObjectHandleResult> _handler;
+        private PriorityQueueObject<T> _head = null;
+        private readonly object _rw_lock = new object();
+        private int _counter = 0;
+
+        public int Count => _counter;
+
+        public ZPriorityQueue(Func<T, PriorityQueueObjectHandleResult> handler)
+        {
+            if (handler == null)
+                throw new ArgumentNullException(nameof(handler));
+            _handler = handler;
+        }
+
+        public void Append(T item, int priority)
+        {
+            var insert = new PriorityQueueObject<T>(item, priority);
+            lock (_rw_lock)
+            {
+                if (null == _head)
+                {
+                    _head = insert;
+                }
+                else
+                {
+                    var cursor = _head;
+                    PriorityQueueObject<T> prev = null;
+                    do
+                    {
+                        if (cursor.Priority > insert.Priority)
+                        {
+                            insert.Next = cursor;
+                            if (null == prev) // insert to head
+                            {
+                                _head = insert;
+                            }
+                            else
+                            {
+                                prev.Next = insert;
+                            }
+                            break;
+                        }
+                        prev = cursor;
+                        cursor = cursor.Next;
+                        if (cursor == null)
+                        {
+                            prev.Next = insert;
+                        }
+                    } while (cursor != null);
+                }
+                _counter++;
+            }
+            return;
+        }
+
+        public T HandleCurrentItem()
+        {
+            T v = default(T);
+            lock (_rw_lock)
+            {
+                var item = _head;
+                PriorityQueueObject<T> prev = null;
+                while (item != null)
+                {
+                    var result = this._handler.Invoke(item.Value);
+                    if (result.IsCompleted)
+                    {
+                        if (prev != null)
+                        {
+                            prev.Next = item.Next;
+                        }
+                        else
+                        {
+                            _head = _head.Next;
+                        }
+                        v = item.Value;
+                        break;
+                    }
+
+                    if (result.CanBeSkipped == false)
+                    {
+                        break;
+                    }
+
+                    prev = item;
+                    item = item.Next;
+                }
+            }
+            return v;
+        }
+    }
+}