diff --git a/Tests/HNSWDemo/HNSWDemo.csproj b/Tests/HNSWDemo/HNSWDemo.csproj index 29cd2e4..46fe152 100644 --- a/Tests/HNSWDemo/HNSWDemo.csproj +++ b/Tests/HNSWDemo/HNSWDemo.csproj @@ -7,7 +7,7 @@ - + diff --git a/Tests/PartitionFileStorageTest/PartitionFileStorageTest.csproj b/Tests/PartitionFileStorageTest/PartitionFileStorageTest.csproj index a75da7d..3d7cb0b 100644 --- a/Tests/PartitionFileStorageTest/PartitionFileStorageTest.csproj +++ b/Tests/PartitionFileStorageTest/PartitionFileStorageTest.csproj @@ -8,7 +8,7 @@ - + diff --git a/Tests/Qdrant.Test/Qdrant.Test.csproj b/Tests/Qdrant.Test/Qdrant.Test.csproj index 2ec3ae0..d459a5e 100644 --- a/Tests/Qdrant.Test/Qdrant.Test.csproj +++ b/Tests/Qdrant.Test/Qdrant.Test.csproj @@ -8,8 +8,8 @@ - - + + diff --git a/Tests/ZeroLevel.UnitTests/PriorityQueueTests.cs b/Tests/ZeroLevel.UnitTests/PriorityQueueTests.cs new file mode 100644 index 0000000..d66ff32 --- /dev/null +++ b/Tests/ZeroLevel.UnitTests/PriorityQueueTests.cs @@ -0,0 +1,105 @@ +using Xunit; +using ZeroLevel.Services.Collections; + +namespace ZeroLevel.UnitTests +{ + public class PriorityQueueTests + { + class TestItem + { + public string Value; + } + + [Fact] + public void AllItemsCorrectHandleTest() + { + // Arrange + var q = new ZPriorityQueue(s => new PriorityQueueObjectHandleResult { IsCompleted = true, CanBeSkipped = false }); + q.Append("s0", 0); + q.Append("s1", 10); + q.Append("s2", 4); + q.Append("s3", 1); + q.Append("s4", 6); + q.Append("s5", 2); + q.Append("s6", 7); + + // Assert + Assert.Equal("s0", q.HandleCurrentItem()); + Assert.Equal("s3", q.HandleCurrentItem()); + Assert.Equal("s5", q.HandleCurrentItem()); + Assert.Equal("s2", q.HandleCurrentItem()); + Assert.Equal("s4", q.HandleCurrentItem()); + Assert.Equal("s6", q.HandleCurrentItem()); + Assert.Equal("s1", q.HandleCurrentItem()); + } + + [Fact] + public void SkipItemsHandleTest() + { + bool skipped = false; + // Arrange + var q = new ZPriorityQueue(s => + { + if (s.Equals("s5") && !skipped) + { + skipped = true; + return new PriorityQueueObjectHandleResult { IsCompleted = false, CanBeSkipped = true }; + } + return new PriorityQueueObjectHandleResult { IsCompleted = true, CanBeSkipped = false }; + }); + q.Append("s0", 0); + q.Append("s1", 10); + q.Append("s2", 4); + q.Append("s3", 1); + q.Append("s4", 6); + q.Append("s5", 2); + q.Append("s6", 7); + + // Assert + Assert.Equal("s0", q.HandleCurrentItem()); + Assert.Equal("s3", q.HandleCurrentItem()); + + Assert.Equal("s2", q.HandleCurrentItem()); + Assert.Equal("s5", q.HandleCurrentItem()); + + Assert.Equal("s4", q.HandleCurrentItem()); + Assert.Equal("s6", q.HandleCurrentItem()); + Assert.Equal("s1", q.HandleCurrentItem()); + } + + [Fact] + public void NoSkipItemsIncorrectHandleTest() + { + bool skipped = false; + // Arrange + var q = new ZPriorityQueue(s => + { + if (s.Equals("s5") && !skipped) + { + skipped = true; + return new PriorityQueueObjectHandleResult { IsCompleted = false, CanBeSkipped = false }; + } + return new PriorityQueueObjectHandleResult { IsCompleted = true, CanBeSkipped = false }; + }); + q.Append("s0", 0); + q.Append("s1", 10); + q.Append("s2", 4); + q.Append("s3", 1); + q.Append("s4", 6); + q.Append("s5", 2); + q.Append("s6", 7); + + // Assert + Assert.Equal("s0", q.HandleCurrentItem()); + Assert.Equal("s3", q.HandleCurrentItem()); + + Assert.Equal(null, q.HandleCurrentItem()); + Assert.Equal("s5", q.HandleCurrentItem()); + Assert.Equal("s2", q.HandleCurrentItem()); + + Assert.Equal("s4", q.HandleCurrentItem()); + Assert.Equal("s6", q.HandleCurrentItem()); + Assert.Equal("s1", q.HandleCurrentItem()); + } + } +} diff --git a/Tests/ZeroLevel.UnitTests/ZeroLevel.UnitTests.csproj b/Tests/ZeroLevel.UnitTests/ZeroLevel.UnitTests.csproj index aa67297..68f40c2 100644 --- a/Tests/ZeroLevel.UnitTests/ZeroLevel.UnitTests.csproj +++ b/Tests/ZeroLevel.UnitTests/ZeroLevel.UnitTests.csproj @@ -9,9 +9,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/ZeroLevel.HNSW/PHNSW/HLevel.cs b/ZeroLevel.HNSW/PHNSW/HLevel.cs new file mode 100644 index 0000000..ad2fc7a --- /dev/null +++ b/ZeroLevel.HNSW/PHNSW/HLevel.cs @@ -0,0 +1,61 @@ +using System; + +namespace ZeroLevel.HNSW.PHNSW +{ + internal class HLevel + : IPHNSWLevel + { + private readonly float _distance; + public HLevel(float distance) + { + _distance = distance; + } + + public Node Node { get; set; } = null; + public IPHNSWLevel NextLevelA { get; set; } + public IPHNSWLevel NextLevelB { get; set; } + + private float _abDistance = float.MinValue; + + public void Add(Node node) + { + if (NextLevelA.Node == null) { NextLevelA.Node = node; } + else if (NextLevelB.Node == null) + { + NextLevelB.Node = node; + _abDistance = PHNSWMetric.CosineDistance(NextLevelA.Node.Vector, NextLevelB.Node.Vector); + } + else + { + var an = PHNSWMetric.CosineDistance(NextLevelA.Node.Vector, node.Vector); + var bn = PHNSWMetric.CosineDistance(NextLevelB.Node.Vector, node.Vector); + + var abDiff = Math.Abs(_distance - _abDistance); + var anDiff = Math.Abs(_distance - an); + var bnDiff = Math.Abs(_distance - bn); + + if (abDiff < anDiff && abDiff < bnDiff) + { + if (an < bn) + { + NextLevelA.Add(node); + } + else + { + NextLevelB.Add(node); + } + } + else if (anDiff < bnDiff && anDiff < abDiff) + { + NextLevelA.Node = node; + NextLevelA.Add(node); + } + else + { + NextLevelB.Node = node; + NextLevelB.Add(node); + } + } + } + } +} diff --git a/ZeroLevel.HNSW/PHNSW/IPHNSWLevel.cs b/ZeroLevel.HNSW/PHNSW/IPHNSWLevel.cs new file mode 100644 index 0000000..64dfba9 --- /dev/null +++ b/ZeroLevel.HNSW/PHNSW/IPHNSWLevel.cs @@ -0,0 +1,8 @@ +namespace ZeroLevel.HNSW.PHNSW +{ + public interface IPHNSWLevel + { + void Add(IPHNSWLevel prevLayer, Node node); + Node Node { get; internal set; } + } +} diff --git a/ZeroLevel.HNSW/PHNSW/Node.cs b/ZeroLevel.HNSW/PHNSW/Node.cs new file mode 100644 index 0000000..1ef51bd --- /dev/null +++ b/ZeroLevel.HNSW/PHNSW/Node.cs @@ -0,0 +1,12 @@ +using ZeroLevel.DocumentObjectModel.Flow; + +namespace ZeroLevel.HNSW.PHNSW +{ + public class Node + { + public float[] Vector { get; set; } + public TPayload Payload { get; set; } + + public List> Neighbors { get; } + } +} diff --git a/ZeroLevel.HNSW/PHNSW/PHNSWBuilder.cs b/ZeroLevel.HNSW/PHNSW/PHNSWBuilder.cs new file mode 100644 index 0000000..6351ba5 --- /dev/null +++ b/ZeroLevel.HNSW/PHNSW/PHNSWBuilder.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ZeroLevel.HNSW.PHNSW +{ + public static class PHNSWBuilder + { + public static IPHNSWLevel Build(int levels) + { + var distance = 0.33f; + var root = new HLevel(distance); + var horizontalLayers = new List>(new[] { root }); + for (var i = 0; i < levels; i++) + { + distance /= 2.0f; + var nextList = new List>(); + foreach (var layer in horizontalLayers) + { + var a = new HLevel(distance); + var b = new HLevel(distance); + layer.NextLevelA = a; + layer.NextLevelB = b; + nextList.Add(a); + nextList.Add(b); + } + horizontalLayers = nextList; + } + var uwLevel = new UWLevel(); + + } + } +} diff --git a/ZeroLevel.HNSW/PHNSW/PHNSWMetric.cs b/ZeroLevel.HNSW/PHNSW/PHNSWMetric.cs new file mode 100644 index 0000000..f6b7dd6 --- /dev/null +++ b/ZeroLevel.HNSW/PHNSW/PHNSWMetric.cs @@ -0,0 +1,28 @@ +using System; + +namespace ZeroLevel.HNSW.PHNSW +{ + internal static class PHNSWMetric + { + internal static float CosineDistance(float[] u, float[] v) + { + if (u.Length != v.Length) + { + throw new ArgumentException("Vectors have non-matching dimensions"); + } + + float dot = 0.0f; + float nru = 0.0f; + float nrv = 0.0f; + for (int i = 0; i < u.Length; ++i) + { + dot += u[i] * v[i]; + nru += u[i] * u[i]; + nrv += v[i] * v[i]; + } + + var similarity = dot / (float)(Math.Sqrt(nru) * Math.Sqrt(nrv)); + return 1 - similarity; + } + } +} diff --git a/ZeroLevel.HNSW/PHNSW/UWLevel.cs b/ZeroLevel.HNSW/PHNSW/UWLevel.cs new file mode 100644 index 0000000..298ac96 --- /dev/null +++ b/ZeroLevel.HNSW/PHNSW/UWLevel.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ZeroLevel.HNSW.PHNSW +{ + public class UWLevel + :IPHNSWLevel + { + } +} diff --git a/ZeroLevel.NN/ZeroLevel.NN.csproj b/ZeroLevel.NN/ZeroLevel.NN.csproj index 396c6ba..c0c8c9a 100644 --- a/ZeroLevel.NN/ZeroLevel.NN.csproj +++ b/ZeroLevel.NN/ZeroLevel.NN.csproj @@ -35,10 +35,10 @@ - - - - + + + + diff --git a/ZeroLevel.Qdrant.GrpcClient/ZeroLevel.Qdrant.GrpcClient.csproj b/ZeroLevel.Qdrant.GrpcClient/ZeroLevel.Qdrant.GrpcClient.csproj index a3c63f0..c1c3b5e 100644 --- a/ZeroLevel.Qdrant.GrpcClient/ZeroLevel.Qdrant.GrpcClient.csproj +++ b/ZeroLevel.Qdrant.GrpcClient/ZeroLevel.Qdrant.GrpcClient.csproj @@ -11,9 +11,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/ZeroLevel.SQL/ZeroLevel.SQL.csproj b/ZeroLevel.SQL/ZeroLevel.SQL.csproj index 27f8cf2..c7e2952 100644 --- a/ZeroLevel.SQL/ZeroLevel.SQL.csproj +++ b/ZeroLevel.SQL/ZeroLevel.SQL.csproj @@ -26,8 +26,8 @@ - - + + diff --git a/ZeroLevel/DataStructures/BitMapCardTable.cs b/ZeroLevel/DataStructures/BitMapCardTable.cs new file mode 100644 index 0000000..f8c8771 --- /dev/null +++ b/ZeroLevel/DataStructures/BitMapCardTable.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; + +namespace ZeroLevel.DataStructures +{ + public sealed class BitMapCardTable + { + private readonly long[] _bitmap; + public BitMapCardTable(int N) + { + var count = N / 64 + (N % 64 == 0 ? 0 : 1); + _bitmap = new long[count]; + } + + public bool this[int index] + { + get + { + if (index < 0) throw new IndexOutOfRangeException(); + var i = index / 64; + var n = _bitmap[i]; + var bit_index = index % 64; + return (n & (1L << bit_index)) != 0; + } + set + { + if (index < 0) throw new IndexOutOfRangeException(); + var i = index / 64; + var bit_index = index % 64; + if (value) + { + _bitmap[i] = _bitmap[i] | (1L << bit_index); + } + else + { + _bitmap[i] = _bitmap[i] & ~(1L << bit_index); + } + } + } + + public void ResetTo(bool value) + { + if (value) + { + for (int i = 0; i < _bitmap.Length; i++) + { + _bitmap[i] = long.MaxValue; + _bitmap[i] |= 1L << 63; + } + } + else + { + for (int i = 0; i < _bitmap.Length; i++) + { + _bitmap[i] = 0; + } + } + } + + public IEnumerable GetSetIndexes() + { + for (int i = 0; i < _bitmap.Length; i++) + { + if (_bitmap[i] != 0) + { + var start = i * 64; + for (int offset = 0; offset < 64; offset++) + { + if ((_bitmap[i] & (1L << offset)) != 0) + { + yield return start + offset; + } + } + } + } + } + } +} diff --git a/ZeroLevel/Services/Collections/ZPriorityQueue.cs b/ZeroLevel/Services/Collections/ZPriorityQueue.cs new file mode 100644 index 0000000..446b4be --- /dev/null +++ b/ZeroLevel/Services/Collections/ZPriorityQueue.cs @@ -0,0 +1,127 @@ +using System; + +namespace ZeroLevel.Services.Collections +{ + public interface IPriorityQueue + { + int Count { get; } + void Append(T item, int priority); + T HandleCurrentItem(); + } + + public struct PriorityQueueObjectHandleResult + { + public bool IsCompleted; + public bool CanBeSkipped; + } + + public class ZPriorityQueue + : IPriorityQueue + { + private sealed class PriorityQueueObject + { + public readonly int Priority; + + public readonly T Value; + + public PriorityQueueObject Next; + + public PriorityQueueObject(T val, int priority) + { + Value = val; + Priority = priority; + } + } + + private readonly Func _handler; + private PriorityQueueObject _head = null; + private readonly object _rw_lock = new object(); + private int _counter = 0; + + public int Count => _counter; + + public ZPriorityQueue(Func handler) + { + if (handler == null) + throw new ArgumentNullException(nameof(handler)); + _handler = handler; + } + + public void Append(T item, int priority) + { + var insert = new PriorityQueueObject(item, priority); + lock (_rw_lock) + { + if (null == _head) + { + _head = insert; + } + else + { + var cursor = _head; + PriorityQueueObject prev = null; + do + { + if (cursor.Priority > insert.Priority) + { + insert.Next = cursor; + if (null == prev) // insert to head + { + _head = insert; + } + else + { + prev.Next = insert; + } + break; + } + prev = cursor; + cursor = cursor.Next; + if (cursor == null) + { + prev.Next = insert; + } + } while (cursor != null); + } + _counter++; + } + return; + } + + public T HandleCurrentItem() + { + T v = default(T); + lock (_rw_lock) + { + var item = _head; + PriorityQueueObject prev = null; + while (item != null) + { + var result = this._handler.Invoke(item.Value); + if (result.IsCompleted) + { + if (prev != null) + { + prev.Next = item.Next; + } + else + { + _head = _head.Next; + } + v = item.Value; + break; + } + + if (result.CanBeSkipped == false) + { + break; + } + + prev = item; + item = item.Next; + } + } + return v; + } + } +}