Apply stashed changes

ZeroLevel.HNSW/Model/SearchContext.cs

@@ -20,12 +20,29 @@ namespace ZeroLevel.HNSW
         private Mode _mode;
 
         public Mode NodeCheckMode => _mode;
+        public double PercentInTotal { get; private set; } = 0;
+        public long AvaliableNodesCount => _activeNodes?.Count ?? 0;
 
         public SearchContext()
         {
             _mode = Mode.None;
         }
 
+        public SearchContext CaclulatePercentage(long total)
+        {
+            if (total > 0)
+            {
+                PercentInTotal = ((_activeNodes.Count * 100d) / (double)total) / 100.0d;
+            }
+            return this;
+        }
+
+        public SearchContext SetPercentage(double percent)
+        {
+            PercentInTotal = percent;
+            return this;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal bool IsActiveNode(int nodeId)
         {

ZeroLevel.HNSW/Services/HNSWMappers.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel.HNSW
@@ -76,6 +77,10 @@ namespace ZeroLevel.HNSW
                         }
                         actives[c].Add(_mappers[c][node]);
                     }
+                    else
+                    {
+                        Log.Warning($"Active node {node} is not included in graphs!");
+                    }
                 }
             }
             if (entryPoints != null)
@@ -91,6 +96,10 @@ namespace ZeroLevel.HNSW
                         }
                         entries[c].Add(_mappers[c][entryPoint]);
                     }
+                    else
+                    {
+                        Log.Warning($"Entry point {entryPoint} is not included in graphs!");
+                    }
                 }
             }
             var result = new Dictionary<int, SearchContext>();
@@ -100,6 +109,23 @@ namespace ZeroLevel.HNSW
                 var entry = entries.GetValueOrDefault(pair.Key);
                 result.Add(pair.Key, new SearchContext().SetActiveNodes(active).SetEntryPointsNodes(entry));
             }
+            var total = result.Values.Sum(v => v.AvaliableNodesCount);
+            if (total > 0)
+            {
+                foreach (var pair in result)
+                {
+                    pair.Value.CaclulatePercentage(total);
+                }
+            }
+            else
+            {
+                //total = result.Values.Sum(v => v.EntryPoints.Count());
+                foreach (var pair in result)
+                {
+                    //var p = (double)pair.Value.EntryPoints.Count() / (double)total;
+                    pair.Value.SetPercentage(0.2d);
+                }
+            }
             return result;
         }
 

ZeroLevel.HNSW/Services/LAL/LALGraph.cs

@@ -91,10 +91,10 @@ namespace ZeroLevel.HNSW
                 int count = reader.ReadInt32(); // Vectors count
                 for (int i = 0; i < count; i++)
                 {
-                    reader.ReadCompatible<TItem>(); //  Vector
+                    var v = reader.ReadCompatible<TItem[]>(); //  Vector
                 }
 
-                reader.ReadInt32(); //  countLayers
+                var lc = reader.ReadInt32(); //  countLayers
                 _links.Deserialize(reader); // deserialize only base layer and skip another
             }
         }

ZeroLevel.HNSW/Services/LAL/SplittedLALGraph.cs

@@ -50,7 +50,6 @@ namespace ZeroLevel.HNSW
 
         public IDictionary<int, List<int>> KNearest(int k, IDictionary<int, SearchContext> contexts)
         {
-            var partial_k = 1 + (k / _graphs.Count);
             var result = new Dictionary<int, List<int>>();
             int step = 1;
             foreach (var graph in _graphs)
@@ -59,12 +58,8 @@ namespace ZeroLevel.HNSW
                 var context = contexts[graph.Key];
                 if (context.EntryPoints != null)
                 {
+                    var partial_k = 1 + (int)(context.PercentInTotal * k);
                     var r = graph.Value.KNearest(partial_k, context) as HashSet<int>;
-                    if (r.Count < partial_k)
-                    {
-                        var diff = partial_k - r.Count;
-                        partial_k += diff / (_graphs.Count - step);
-                    }
                     result[graph.Key].AddRange(r);
                 }
                 step++;

ZeroLevel/Services/FileSystem/FSUtils.cs

@@ -243,6 +243,31 @@ namespace ZeroLevel.Services.FileSystem
             return new string(result, 0, index);
         }
 
+        /// <summary>
+        /// Removes invalid characters from the passed file name
+        /// </summary>
+        /// <param name="path"></param>
+        /// <returns></returns>
+        public static string FileNameCorrection(string path, char replaceChar)
+        {
+            if (path == null) return string.Empty;
+            var result = new char[path.Length];
+            var index = 0;
+            foreach (char c in path)
+            {
+                if (_invalid_filename_characters.IndexOf(c) >= 0)
+                {
+                    result[index] = replaceChar;
+                }
+                else
+                {
+                    result[index] = c;
+                }
+                index++;
+            }
+            return new string(result, 0, index);
+        }
+
         #endregion FileName & Path correction
 
         /// <summary>

ZeroLevel/Services/Reflection/TypeHelpers.cs

@@ -27,6 +27,26 @@ namespace ZeroLevel.Services.Reflection
             if (baseType == null) return false;
             return IsAssignableToGenericType(baseType, genericType);
         }
+        public static bool IsNumericType(Type type)
+        {
+            switch (Type.GetTypeCode(type))
+            {
+                case TypeCode.Byte:
+                case TypeCode.SByte:
+                case TypeCode.UInt16:
+                case TypeCode.UInt32:
+                case TypeCode.UInt64:
+                case TypeCode.Int16:
+                case TypeCode.Int32:
+                case TypeCode.Int64:
+                case TypeCode.Decimal:
+                case TypeCode.Double:
+                case TypeCode.Single:
+                    return true;
+                default:
+                    return false;
+            }
+        }
 
         public static bool IsNumericType(Type type)
         {