diff --git a/ZeroLevel.HNSW/Model/SearchContext.cs b/ZeroLevel.HNSW/Model/SearchContext.cs index 5a0bf4b..d9058a4 100644 --- a/ZeroLevel.HNSW/Model/SearchContext.cs +++ b/ZeroLevel.HNSW/Model/SearchContext.cs @@ -20,12 +20,29 @@ namespace ZeroLevel.HNSW private Mode _mode; public Mode NodeCheckMode => _mode; + public double PercentInTotal { get; private set; } = 0; + public long AvaliableNodesCount => _activeNodes?.Count ?? 0; public SearchContext() { _mode = Mode.None; } + public SearchContext CaclulatePercentage(long total) + { + if (total > 0) + { + PercentInTotal = ((_activeNodes.Count * 100d) / (double)total) / 100.0d; + } + return this; + } + + public SearchContext SetPercentage(double percent) + { + PercentInTotal = percent; + return this; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool IsActiveNode(int nodeId) { diff --git a/ZeroLevel.HNSW/Services/HNSWMappers.cs b/ZeroLevel.HNSW/Services/HNSWMappers.cs index 1f124c0..8ac018e 100644 --- a/ZeroLevel.HNSW/Services/HNSWMappers.cs +++ b/ZeroLevel.HNSW/Services/HNSWMappers.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using ZeroLevel.Services.Serialization; namespace ZeroLevel.HNSW @@ -76,6 +77,10 @@ namespace ZeroLevel.HNSW } actives[c].Add(_mappers[c][node]); } + else + { + Log.Warning($"Active node {node} is not included in graphs!"); + } } } if (entryPoints != null) @@ -91,6 +96,10 @@ namespace ZeroLevel.HNSW } entries[c].Add(_mappers[c][entryPoint]); } + else + { + Log.Warning($"Entry point {entryPoint} is not included in graphs!"); + } } } var result = new Dictionary(); @@ -100,6 +109,23 @@ namespace ZeroLevel.HNSW var entry = entries.GetValueOrDefault(pair.Key); result.Add(pair.Key, new SearchContext().SetActiveNodes(active).SetEntryPointsNodes(entry)); } + var total = result.Values.Sum(v => v.AvaliableNodesCount); + if (total > 0) + { + foreach (var pair in result) + { + pair.Value.CaclulatePercentage(total); + } + } + else + { + //total = result.Values.Sum(v => v.EntryPoints.Count()); + foreach (var pair in result) + { + //var p = (double)pair.Value.EntryPoints.Count() / (double)total; + pair.Value.SetPercentage(0.2d); + } + } return result; } diff --git a/ZeroLevel.HNSW/Services/LAL/LALGraph.cs b/ZeroLevel.HNSW/Services/LAL/LALGraph.cs index ea0a0e1..622f185 100644 --- a/ZeroLevel.HNSW/Services/LAL/LALGraph.cs +++ b/ZeroLevel.HNSW/Services/LAL/LALGraph.cs @@ -91,10 +91,10 @@ namespace ZeroLevel.HNSW int count = reader.ReadInt32(); // Vectors count for (int i = 0; i < count; i++) { - reader.ReadCompatible(); // Vector + var v = reader.ReadCompatible(); // Vector } - reader.ReadInt32(); // countLayers + var lc = reader.ReadInt32(); // countLayers _links.Deserialize(reader); // deserialize only base layer and skip another } } diff --git a/ZeroLevel.HNSW/Services/LAL/SplittedLALGraph.cs b/ZeroLevel.HNSW/Services/LAL/SplittedLALGraph.cs index 243cf99..3c2ce70 100644 --- a/ZeroLevel.HNSW/Services/LAL/SplittedLALGraph.cs +++ b/ZeroLevel.HNSW/Services/LAL/SplittedLALGraph.cs @@ -50,7 +50,6 @@ namespace ZeroLevel.HNSW public IDictionary> KNearest(int k, IDictionary contexts) { - var partial_k = 1 + (k / _graphs.Count); var result = new Dictionary>(); int step = 1; foreach (var graph in _graphs) @@ -59,12 +58,8 @@ namespace ZeroLevel.HNSW var context = contexts[graph.Key]; if (context.EntryPoints != null) { + var partial_k = 1 + (int)(context.PercentInTotal * k); var r = graph.Value.KNearest(partial_k, context) as HashSet; - if (r.Count < partial_k) - { - var diff = partial_k - r.Count; - partial_k += diff / (_graphs.Count - step); - } result[graph.Key].AddRange(r); } step++; diff --git a/ZeroLevel/Services/FileSystem/FSUtils.cs b/ZeroLevel/Services/FileSystem/FSUtils.cs index 3db7ff4..fe0df44 100644 --- a/ZeroLevel/Services/FileSystem/FSUtils.cs +++ b/ZeroLevel/Services/FileSystem/FSUtils.cs @@ -243,6 +243,31 @@ namespace ZeroLevel.Services.FileSystem return new string(result, 0, index); } + /// + /// Removes invalid characters from the passed file name + /// + /// + /// + public static string FileNameCorrection(string path, char replaceChar) + { + if (path == null) return string.Empty; + var result = new char[path.Length]; + var index = 0; + foreach (char c in path) + { + if (_invalid_filename_characters.IndexOf(c) >= 0) + { + result[index] = replaceChar; + } + else + { + result[index] = c; + } + index++; + } + return new string(result, 0, index); + } + #endregion FileName & Path correction /// diff --git a/ZeroLevel/Services/Reflection/TypeHelpers.cs b/ZeroLevel/Services/Reflection/TypeHelpers.cs index 87625b2..5b77e4a 100644 --- a/ZeroLevel/Services/Reflection/TypeHelpers.cs +++ b/ZeroLevel/Services/Reflection/TypeHelpers.cs @@ -27,6 +27,26 @@ namespace ZeroLevel.Services.Reflection if (baseType == null) return false; return IsAssignableToGenericType(baseType, genericType); } + public static bool IsNumericType(Type type) + { + switch (Type.GetTypeCode(type)) + { + case TypeCode.Byte: + case TypeCode.SByte: + case TypeCode.UInt16: + case TypeCode.UInt32: + case TypeCode.UInt64: + case TypeCode.Int16: + case TypeCode.Int32: + case TypeCode.Int64: + case TypeCode.Decimal: + case TypeCode.Double: + case TypeCode.Single: + return true; + default: + return false; + } + } public static bool IsNumericType(Type type) {