ogoun
/
Zero
mirror of https://github.com/ogoun/Zero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

HNSW

Browse Source 
Links trimming
pull/1/head
unknown 3 years ago
parent 50e2e90251
commit 1b6c56033f
9 changed files with 100 additions and 44 deletions
Show Stats Download Patch File Download Diff File

10
TestHNSW/HNSWDemo/Program.cs
Unescape View File

@ -165,7 +165,7 @@ namespace HNSWDemo
static void Main(string[] args) static void Main(string[] args)
{ {
AccuracityTest(); InsertTimeExplosionTest();
Console.WriteLine("Completed"); Console.WriteLine("Completed");
Console.ReadKey(); Console.ReadKey();
} }
@ -661,18 +661,18 @@ namespace HNSWDemo
static void InsertTimeExplosionTest() static void InsertTimeExplosionTest()
{ {
var count = 1000; var count = 20000;
var iterationCount = 1000; var iterationCount = 100;
var dimensionality = 128; var dimensionality = 128;
var sw = new Stopwatch(); var sw = new Stopwatch();
var world = new SmallWorld<float[]>(NSWOptions<float[]>.Create(6, 12, 100, 100, CosineDistance.NonOptimized, true, true, selectionHeuristic: NeighbourSelectionHeuristic.SelectSimple)); var world = new SmallWorld<float[]>(NSWOptions<float[]>.Create(6, 8, 150, 150, Metrics.L2Euclidean, true, true, selectionHeuristic: NeighbourSelectionHeuristic.SelectSimple));
for (int i = 0; i < iterationCount; i++) for (int i = 0; i < iterationCount; i++)
{ {
var samples = RandomVectors(dimensionality, count); var samples = RandomVectors(dimensionality, count);
sw.Restart(); sw.Restart();
var ids = world.AddItems(samples.ToArray()); var ids = world.AddItems(samples.ToArray());
sw.Stop(); sw.Stop();
Console.WriteLine($"ITERATION: [{i.ToString("D4")}] COUNT: [{ids.Length}] ELAPSEF [{sw.ElapsedMilliseconds} ms]"); Console.WriteLine($"ITERATION: [{i.ToString("D4")}] COUNT: [{ids.Length}] ELAPSED [{sw.ElapsedMilliseconds} ms]");
} }
} }
} }

22
ZeroLevel.HNSW/Services/CompactBiDirectionalLinksSet.cs
Unescape View File

@ -184,6 +184,28 @@ namespace ZeroLevel.HNSW
} }
} }
internal void RemoveIndex(int id1, int id2)
{
long k1 = (((long)(id1)) << HALF_LONG_BITS) + id2;
long k2 = (((long)(id2)) << HALF_LONG_BITS) + id1;
_rwLock.EnterWriteLock();
try
{
if (_set.ContainsKey(k1))
{
_set.Remove(k1);
}
if (_set.ContainsKey(k2))
{
_set.Remove(k2);
}
}
finally
{
_rwLock.ExitWriteLock();
}
}
internal bool Add(int id1, int id2, float distance) internal bool Add(int id1, int id2, float distance)
{ {
_rwLock.EnterWriteLock(); _rwLock.EnterWriteLock();

48
ZeroLevel.HNSW/Services/Layer.cs
Unescape View File

@ -41,7 +41,7 @@ namespace ZeroLevel.HNSW
/// <param name="p">The node with which the connection will be made</param> /// <param name="p">The node with which the connection will be made</param>
/// <param name="qpDistance"></param> /// <param name="qpDistance"></param>
/// <param name="isMapLayer"></param> /// <param name="isMapLayer"></param>
internal void AddBidirectionallConnections(int q, int p, float qpDistance, bool isMapLayer) /*internal void AddBidirectionallConnections(int q, int p, float qpDistance, bool isMapLayer)
{ {
// поиск в ширину ближайших узлов к найденному // поиск в ширину ближайших узлов к найденному
var nearest = _links.FindLinksForId(p).ToArray(); var nearest = _links.FindLinksForId(p).ToArray();
@ -86,8 +86,40 @@ namespace ZeroLevel.HNSW
_links.Add(q, p, qpDistance); _links.Add(q, p, qpDistance);
} }
} }
}*/
internal void AddBidirectionallConnections(int q, int p, float qpDistance)
{
_links.Add(q, p, qpDistance);
} }
internal void TrimLinks(int q, bool isMapLayer)
{
var M = (isMapLayer ? _options.M * 2 : _options.M);
// поиск в ширину ближайших узлов к найденному
var nearest = _links.FindLinksForId(q).ToArray();
if (nearest.Length <= M && nearest.Length > 1)
{
foreach (var l in nearest)
{
if (l.Item1 == l.Item2)
{
_links.RemoveIndex(l.Item1, l.Item2);
}
}
}
else if (nearest.Length > M)
{
var removeCount = nearest.Length - M;
foreach (var l in nearest.OrderByDescending(n => n.Item3).Take(removeCount))
{
_links.RemoveIndex(l.Item1, l.Item2);
}
}
}
/// <summary> /// <summary>
/// Adding a node with a connection to itself /// Adding a node with a connection to itself
/// </summary> /// </summary>
@ -251,12 +283,12 @@ namespace ZeroLevel.HNSW
var toExpand = C.Pop(); var toExpand = C.Pop();
if (W.Count > 0) if (W.Count > 0)
{ {
if(W.TryPeek(out _, out var dist )) if (W.TryPeek(out _, out var dist))
if (toExpand.Item2 > dist) if (toExpand.Item2 > dist)
{ {
// the closest candidate is farther than farthest result // the closest candidate is farther than farthest result
break; break;
} }
} }
// expand candidate // expand candidate
@ -270,7 +302,7 @@ namespace ZeroLevel.HNSW
var neighbourDistance = targetCosts(neighbourId); var neighbourDistance = targetCosts(neighbourId);
if (context.IsActiveNode(neighbourId)) if (context.IsActiveNode(neighbourId))
{ {
if (W.Count < ef || (W.Count > 0 && (W.TryPeek(out _, out var dist) && neighbourDistance < dist))) if (W.Count < ef || (W.Count > 0 && (W.TryPeek(out _, out var dist) && neighbourDistance < dist)))
{ {
W.Push((neighbourId, neighbourDistance)); W.Push((neighbourId, neighbourDistance));
if (W.Count > ef) if (W.Count > ef)

7
ZeroLevel.HNSW/SmallWorld.cs
Unescape View File

@ -119,7 +119,7 @@ namespace ZeroLevel.HNSW
var W = new MinHeap(_options.EFConstruction + 1); var W = new MinHeap(_options.EFConstruction + 1);
// ep ← get enter point for hnsw // ep ← get enter point for hnsw
var ep = _layers[MaxLayer].FindEntryPointAtLayer(distance); var ep = _layers[MaxLayer].FindEntryPointAtLayer(distance);
if(ep == -1) ep = EntryPoint; if (ep == -1) ep = EntryPoint;
var epDist = distance(ep); var epDist = distance(ep);
// L ← level of ep // top layer for hnsw // L ← level of ep // top layer for hnsw
var L = MaxLayer; var L = MaxLayer;
@ -143,6 +143,7 @@ namespace ZeroLevel.HNSW
ep = id; ep = id;
epDist = value; epDist = value;
} }
_layers[lc].TrimLinks(q, lc == 0);
W.Clear(); W.Clear();
} }
//for lc ← min(L, l) … 0 //for lc ← min(L, l) … 0
@ -175,7 +176,7 @@ namespace ZeroLevel.HNSW
foreach (var e in neighbors) foreach (var e in neighbors)
{ {
// eConn ← neighbourhood(e) at layer lc // eConn ← neighbourhood(e) at layer lc
_layers[lc].AddBidirectionallConnections(q, e.Item1, e.Item2, lc == 0); _layers[lc].AddBidirectionallConnections(q, e.Item1, e.Item2);
// if distance from newNode to newNeighbour is better than to bestPeer => update bestPeer // if distance from newNode to newNeighbour is better than to bestPeer => update bestPeer
if (e.Item2 < epDist) if (e.Item2 < epDist)
{ {
@ -183,6 +184,8 @@ namespace ZeroLevel.HNSW
epDist = e.Item2; epDist = e.Item2;
} }
} }
_layers[lc].TrimLinks(q, lc == 0);
W.Clear(); W.Clear();
} }
} }

10
ZeroLevel/Services/DOM/DSL/Services/TContainerFactory.cs
Unescape View File

@ -5,20 +5,20 @@ namespace DOM.DSL.Services
{ {
public class TContainerFactory public class TContainerFactory
{ {
private readonly ObjectPool<TContainer> _pool; private readonly Pool<TContainer> _pool;
private static int _get_count = 0; private static int _get_count = 0;
private static int _release_count = 0; private static int _release_count = 0;
internal TContainerFactory(TRender render) internal TContainerFactory(TRender render)
{ {
_pool = new ObjectPool<TContainer>(() => new TContainer(this, render), 64); _pool = new Pool<TContainer>(64, p => new TContainer(this, render));
} }
internal TContainer Get(object value) internal TContainer Get(object value)
{ {
Interlocked.Increment(ref _get_count); Interlocked.Increment(ref _get_count);
var c = _pool.Allocate(); var c = _pool.Acquire();
c.Reset(value); c.Reset(value);
return c; return c;
} }
@ -26,7 +26,7 @@ namespace DOM.DSL.Services
internal TContainer Get(object value, int index) internal TContainer Get(object value, int index)
{ {
Interlocked.Increment(ref _get_count); Interlocked.Increment(ref _get_count);
var c = _pool.Allocate(); var c = _pool.Acquire();
c.Reset(value); c.Reset(value);
c.Index = index; c.Index = index;
return c; return c;
@ -37,7 +37,7 @@ namespace DOM.DSL.Services
if (container != null) if (container != null)
{ {
Interlocked.Increment(ref _release_count); Interlocked.Increment(ref _release_count);
_pool.Free(container); _pool.Release(container);
} }
} }

34
ZeroLevel/Services/Network/Extensions/ExchangeExtension.cs
Unescape View File

@ -6,12 +6,12 @@ namespace ZeroLevel.Network
{ {
public static class ExchangeExtension public static class ExchangeExtension
{ {
static ObjectPool<AutoResetEvent> _mrePool = new ObjectPool<AutoResetEvent>(() => new AutoResetEvent(false), 16); static Pool<AutoResetEvent> _mrePool = new Pool<AutoResetEvent>(16, (p) => new AutoResetEvent(false));
public static Tresponse Request<Tresponse>(this IClientSet exchange, string alias, TimeSpan timeout) public static Tresponse Request<Tresponse>(this IClientSet exchange, string alias, TimeSpan timeout)
{ {
Tresponse response = default; Tresponse response = default;
var ev = _mrePool.Allocate(); var ev = _mrePool.Acquire();
try try
{ {
if (exchange.Request<Tresponse>(alias, if (exchange.Request<Tresponse>(alias,
@ -22,7 +22,7 @@ namespace ZeroLevel.Network
} }
finally finally
{ {
_mrePool.Free(ev); _mrePool.Release(ev);
} }
return response; return response;
} }
@ -30,7 +30,7 @@ namespace ZeroLevel.Network
public static Tresponse Request<Tresponse>(this IClientSet exchange, string alias, string inbox, TimeSpan timeout) public static Tresponse Request<Tresponse>(this IClientSet exchange, string alias, string inbox, TimeSpan timeout)
{ {
Tresponse response = default; Tresponse response = default;
var ev = _mrePool.Allocate(); var ev = _mrePool.Acquire();
try try
{ {
if (exchange.Request<Tresponse>(alias, inbox, if (exchange.Request<Tresponse>(alias, inbox,
@ -44,7 +44,7 @@ namespace ZeroLevel.Network
} }
finally finally
{ {
_mrePool.Free(ev); _mrePool.Release(ev);
} }
return response; return response;
} }
@ -52,7 +52,7 @@ namespace ZeroLevel.Network
public static Tresponse Request<Trequest, Tresponse>(this IClientSet exchange, string alias, Trequest request, TimeSpan timeout) public static Tresponse Request<Trequest, Tresponse>(this IClientSet exchange, string alias, Trequest request, TimeSpan timeout)
{ {
Tresponse response = default; Tresponse response = default;
var ev = _mrePool.Allocate(); var ev = _mrePool.Acquire();
try try
{ {
if (exchange.Request<Trequest, Tresponse>(alias, request, if (exchange.Request<Trequest, Tresponse>(alias, request,
@ -63,7 +63,7 @@ namespace ZeroLevel.Network
} }
finally finally
{ {
_mrePool.Free(ev); _mrePool.Release(ev);
} }
return response; return response;
} }
@ -72,7 +72,7 @@ namespace ZeroLevel.Network
, Trequest request, TimeSpan timeout) , Trequest request, TimeSpan timeout)
{ {
Tresponse response = default; Tresponse response = default;
var ev = _mrePool.Allocate(); var ev = _mrePool.Acquire();
try try
{ {
if (exchange.Request<Trequest, Tresponse>(alias, inbox, request, if (exchange.Request<Trequest, Tresponse>(alias, inbox, request,
@ -83,7 +83,7 @@ namespace ZeroLevel.Network
} }
finally finally
{ {
_mrePool.Free(ev); _mrePool.Release(ev);
} }
return response; return response;
} }
@ -91,7 +91,7 @@ namespace ZeroLevel.Network
public static Tresponse Request<Tresponse>(this IClientSet exchange, string alias) public static Tresponse Request<Tresponse>(this IClientSet exchange, string alias)
{ {
Tresponse response = default; Tresponse response = default;
var ev = _mrePool.Allocate(); var ev = _mrePool.Acquire();
try try
{ {
if (exchange.Request<Tresponse>(alias, if (exchange.Request<Tresponse>(alias,
@ -102,7 +102,7 @@ namespace ZeroLevel.Network
} }
finally finally
{ {
_mrePool.Free(ev); _mrePool.Release(ev);
} }
return response; return response;
} }
@ -110,7 +110,7 @@ namespace ZeroLevel.Network
public static Tresponse Request<Tresponse>(this IClientSet exchange, string alias, string inbox) public static Tresponse Request<Tresponse>(this IClientSet exchange, string alias, string inbox)
{ {
Tresponse response = default; Tresponse response = default;
var ev = _mrePool.Allocate(); var ev = _mrePool.Acquire();
try try
{ {
if (exchange.Request<Tresponse>(alias, inbox, if (exchange.Request<Tresponse>(alias, inbox,
@ -124,7 +124,7 @@ namespace ZeroLevel.Network
} }
finally finally
{ {
_mrePool.Free(ev); _mrePool.Release(ev);
} }
return response; return response;
} }
@ -132,7 +132,7 @@ namespace ZeroLevel.Network
public static Tresponse Request<Trequest, Tresponse>(this IClientSet exchange, string alias, Trequest request) public static Tresponse Request<Trequest, Tresponse>(this IClientSet exchange, string alias, Trequest request)
{ {
Tresponse response = default; Tresponse response = default;
var ev = _mrePool.Allocate(); var ev = _mrePool.Acquire();
try try
{ {
if (exchange.Request<Trequest, Tresponse>(alias, request, if (exchange.Request<Trequest, Tresponse>(alias, request,
@ -143,7 +143,7 @@ namespace ZeroLevel.Network
} }
finally finally
{ {
_mrePool.Free(ev); _mrePool.Release(ev);
} }
return response; return response;
} }
@ -152,7 +152,7 @@ namespace ZeroLevel.Network
, Trequest request) , Trequest request)
{ {
Tresponse response = default; Tresponse response = default;
var ev = _mrePool.Allocate(); var ev = _mrePool.Acquire();
try try
{ {
if (exchange.Request<Trequest, Tresponse>(alias, inbox, request, if (exchange.Request<Trequest, Tresponse>(alias, inbox, request,
@ -163,7 +163,7 @@ namespace ZeroLevel.Network
} }
finally finally
{ {
_mrePool.Free(ev); _mrePool.Release(ev);
} }
return response; return response;
} }

6
ZeroLevel/Services/Network/FileTransfer/FileSender.cs
Unescape View File

@ -10,7 +10,7 @@ namespace ZeroLevel.Network.FileTransfer
public sealed class FileSender public sealed class FileSender
{ {
private BlockingCollection<FileTransferTask> _tasks = new BlockingCollection<FileTransferTask>(); private BlockingCollection<FileTransferTask> _tasks = new BlockingCollection<FileTransferTask>();
private ObjectPool<FileTransferTask> _taskPool = new ObjectPool<FileTransferTask>(() => new FileTransferTask(), 100); private Pool<FileTransferTask> _taskPool = new Pool<FileTransferTask>(100, (p) => new FileTransferTask());
private readonly Thread _uploadFileThread; private readonly Thread _uploadFileThread;
private bool _resendWhenServerError = false; private bool _resendWhenServerError = false;
private bool _resendWhenClientError = false; private bool _resendWhenClientError = false;
@ -45,7 +45,7 @@ namespace ZeroLevel.Network.FileTransfer
{ {
throw new FileNotFoundException(filePath); throw new FileNotFoundException(filePath);
} }
var task = _taskPool.Allocate(); var task = _taskPool.Acquire();
task.CompletedHandler = completeHandler; task.CompletedHandler = completeHandler;
task.ErrorHandler = errorHandler; task.ErrorHandler = errorHandler;
task.FilePath = filePath; task.FilePath = filePath;
@ -69,7 +69,7 @@ namespace ZeroLevel.Network.FileTransfer
} }
finally finally
{ {
_taskPool.Free(task); _taskPool.Release(task);
} }
} }
} }

6
ZeroLevel/Services/Semantic/WordTokenizer.cs
Unescape View File

@ -6,13 +6,13 @@ namespace ZeroLevel.Services.Semantic
{ {
public static class WordTokenizer public static class WordTokenizer
{ {
static ObjectPool<char[]> _pool = new ObjectPool<char[]>(() => new char[2048]); static Pool<char[]> _pool = new Pool<char[]>(64 ,(p) => new char[2048]);
public static IEnumerable<string> Tokenize(string text) public static IEnumerable<string> Tokenize(string text)
{ {
int index = 0; int index = 0;
bool first = true; bool first = true;
var buffer = _pool.Allocate(); var buffer = _pool.Acquire();
try try
{ {
for (int i = 0; i < text?.Length; i++) for (int i = 0; i < text?.Length; i++)
@ -40,7 +40,7 @@ namespace ZeroLevel.Services.Semantic
} }
finally finally
{ {
_pool.Free(buffer); _pool.Release(buffer);
} }
} }
} }

1
temp2/Program.cs
Unescape View File

@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using ZeroLevel.HNSW; using ZeroLevel.HNSW;
using ZeroLevel.HNSW.Services.OPT;
using ZeroLevel.Services.Serialization; using ZeroLevel.Services.Serialization;
namespace temp2 namespace temp2

Write Preview
Loading…
Cancel
Save

Powered by TurnKey Linux.