HNSW graphs serialization

TestHNSW/HNSWDemo/Tests/LALTest.cs

@@ -12,6 +12,8 @@ namespace HNSWDemo.Tests
     {
         private const int count = 20000;
         private const int dimensionality = 128;
+        private const string _graphFileCachee = @"lal_test_graph.bin";
+        private const string _mapFileCachee = @"lal_test_map.bin";
 
         public void Run()
         {
@@ -27,9 +29,19 @@ namespace HNSWDemo.Tests
                 samples[c].Add(p);
             }
 
+            SplittedLALGraph worlds;
+            HNSWMappers<long> mappers;
 
-            var worlds = new SplittedLALGraph();
-            var mappers = new HNSWMappers<long>(l => (int)Math.Abs(l.GetHashCode() % moda));
+            if (File.Exists(_graphFileCachee) && File.Exists(_mapFileCachee))
+            {
+                worlds = new SplittedLALGraph(_graphFileCachee);
+                mappers = new HNSWMappers<long>(_mapFileCachee, l => (int)Math.Abs(l.GetHashCode() % moda));
+            }
+            else
+            {
+
+                worlds = new SplittedLALGraph();
+                mappers = new HNSWMappers<long>(l => (int)Math.Abs(l.GetHashCode() % moda));
 
                 var worlds_dict = new Dictionary<int, SmallWorld<float[]>>();
                 var maps_dict = new Dictionary<int, HNSWMap<long>>();
@@ -75,12 +87,15 @@ namespace HNSWDemo.Tests
                     mappers.Append(m, c);
                 }
 
+                worlds.Save(_graphFileCachee);
+                mappers.Save(_mapFileCachee);
+            }
+
             var entries = new long[10];
             for (int i = 0; i < entries.Length; i++)
             {
                 entries[i] = persons[DefaultRandomGenerator.Instance.Next(0, persons.Count - 1)].Item2.Number;
             }
-
             var contexts = mappers.CreateContext(null, entries);
             var result = worlds.KNearest(5000, contexts);
 

ZeroLevel.HNSW/Services/HNSWMap.cs

@@ -14,6 +14,8 @@ namespace ZeroLevel.HNSW
         private Dictionary<int, TFeature> _reverse_map;
 
         public int this[TFeature feature] => _map.GetValueOrDefault(feature);
+
+        public HNSWMap() { }
         public HNSWMap(int capacity = -1)
         {
             if (capacity > 0)

ZeroLevel.HNSW/Services/HNSWMappers.cs

@@ -1,14 +1,48 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
+using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel.HNSW
 {
     public class HNSWMappers<TFeature>
+        : IBinarySerializable
     {
-        private readonly IDictionary<int, HNSWMap<TFeature>> _mappers = new Dictionary<int, HNSWMap<TFeature>>();
+        private IDictionary<int, HNSWMap<TFeature>> _mappers;
         private readonly Func<TFeature, int> _bucketFunction;
+
+        public HNSWMappers(string filePath, Func<TFeature, int> bucketFunction)
+        {
+            _bucketFunction = bucketFunction;
+            using (var fs = File.OpenRead(filePath))
+            {
+                using (var bs = new BufferedStream(fs, 1024 * 1024 * 32))
+                {
+                    using (var reader = new MemoryStreamReader(bs))
+                    {
+                        Deserialize(reader);
+                    }
+                }
+            }
+        }
+
+        public void Save(string filePath)
+        {
+            using (var fs = File.OpenWrite(filePath))
+            {
+                using (var bs = new BufferedStream(fs, 1024 * 1024 * 32))
+                {
+                    using (var writer = new MemoryStreamWriter(bs))
+                    {
+                        Serialize(writer);
+                    }
+                }
+            }
+        }
+
         public HNSWMappers(Func<TFeature, int> bucketFunction)
         {
+            _mappers = new Dictionary<int, HNSWMap<TFeature>>();
             _bucketFunction = bucketFunction;
         }
 
@@ -68,5 +102,15 @@ namespace ZeroLevel.HNSW
             }
             return result;
         }
+
+        public void Deserialize(IBinaryReader reader)
+        {
+            this._mappers = reader.ReadDictionary<int, HNSWMap<TFeature>>();
+        }
+
+        public void Serialize(IBinaryWriter writer)
+        {
+            writer.WriteDictionary<int, HNSWMap<TFeature>>(this._mappers);
+        }
     }
 }

ZeroLevel.HNSW/Services/LAL/LALGraph.cs

@@ -6,10 +6,11 @@ using ZeroLevel.Services.Serialization;
 namespace ZeroLevel.HNSW
 {
     public class LALGraph
+        : IBinarySerializable
     {
         private readonly LALLinks _links = new LALLinks();
 
-        private LALGraph() { }
+        public LALGraph() { }
         public static LALGraph FromLALGraph(Stream stream)
         {
             var l = new LALGraph();
@@ -76,7 +77,7 @@ namespace ZeroLevel.HNSW
         {
             using (var reader = new MemoryStreamReader(stream))
             {
-                _links.Deserialize(reader); // deserialize only base layer and skip another
+                _links.Deserialize(reader);
             }
         }
 
@@ -105,5 +106,15 @@ namespace ZeroLevel.HNSW
                 _links.Serialize(writer);
             }
         }
+
+        public void Serialize(IBinaryWriter writer)
+        {
+            _links.Serialize(writer);
+        }
+
+        public void Deserialize(IBinaryReader reader)
+        {
+            _links.Deserialize(reader);
+        }
     }
 }

ZeroLevel.HNSW/Services/LAL/LALLinks.cs

@@ -6,6 +6,7 @@ using ZeroLevel.Services.Serialization;
 namespace ZeroLevel.HNSW
 {
     internal class LALLinks
+        : IBinarySerializable
     {
         private ConcurrentDictionary<int, int[]> _set = new ConcurrentDictionary<int, int[]>();
         internal IDictionary<int, int[]> Links => _set;
@@ -47,6 +48,7 @@ namespace ZeroLevel.HNSW
             _set.Clear();
             _set = null;
         }
+
         public void Serialize(IBinaryWriter writer)
         {
             writer.WriteInt32(_set.Count);

ZeroLevel.HNSW/Services/LAL/SplittedLALGraph.cs

@@ -1,10 +1,47 @@
 using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using ZeroLevel.Services.Serialization;
 
 namespace ZeroLevel.HNSW
 {
     public class SplittedLALGraph
+        : IBinarySerializable
     {
-        private readonly IDictionary<int, LALGraph> _graphs = new Dictionary<int, LALGraph>();
+        private IDictionary<int, LALGraph> _graphs;
+
+        public SplittedLALGraph()
+        {
+            _graphs = new Dictionary<int, LALGraph>();
+        }
+
+        public SplittedLALGraph(string filePath)
+        {
+            using (var fs = File.OpenRead(filePath))
+            {
+                using (var bs = new BufferedStream(fs, 1024 * 1024 * 32))
+                {
+                    using (var reader = new MemoryStreamReader(bs))
+                    {
+                        Deserialize(reader);
+                    }
+                }
+            }
+        }
+
+        public void Save(string filePath)
+        {
+            using (var fs = File.OpenWrite(filePath))
+            {
+                using (var bs = new BufferedStream(fs, 1024 * 1024 * 32))
+                {
+                    using (var writer = new MemoryStreamWriter(bs))
+                    {
+                        Serialize(writer);
+                    }
+                }
+            }
+        }
 
         public void Append(LALGraph graph, int c)
         {
@@ -34,5 +71,14 @@ namespace ZeroLevel.HNSW
             }
             return result;
         }
+
+        public void Serialize(IBinaryWriter writer)
+        {
+            writer.WriteDictionary<int, LALGraph>(this._graphs);
+        }
+        public void Deserialize(IBinaryReader reader)
+        {
+            this._graphs = reader.ReadDictionary<int, LALGraph>();
+        }
     }
 }

ZeroLevel/Services/Serialization/IBinaryWriter.cs

@@ -96,7 +96,7 @@ namespace ZeroLevel.Services.Serialization
         void WriteCollection(IEnumerable<IPAddress> collection);
         #endregion
 
-        void WriteDictionary<TKey, TValue>(Dictionary<TKey, TValue> collection);
+        void WriteDictionary<TKey, TValue>(IDictionary<TKey, TValue> collection);
         void WriteDictionary<TKey, TValue>(ConcurrentDictionary<TKey, TValue> collection);
 
         void Write<T>(T item)

ZeroLevel/Services/Serialization/MemoryStreamWriter.cs

@@ -777,7 +777,7 @@ namespace ZeroLevel.Services.Serialization
             }
         }
 
-        public void WriteDictionary<TKey, TValue>(Dictionary<TKey, TValue> collection)
+        public void WriteDictionary<TKey, TValue>(IDictionary<TKey, TValue> collection)
         {
             WriteInt32(collection?.Count() ?? 0);
             if (collection != null)