diff --git a/ZeroLevel.HNSW/Model/SearchContext.cs b/ZeroLevel.HNSW/Model/SearchContext.cs
index d9058a4..a80597c 100644
--- a/ZeroLevel.HNSW/Model/SearchContext.cs
+++ b/ZeroLevel.HNSW/Model/SearchContext.cs
@@ -15,8 +15,17 @@ namespace ZeroLevel.HNSW
public sealed class SearchContext
{
+ ///
+ /// Список номеров которые разрешены к добавлению итогового результата, если поиск ведется в ограниченном наборе точек (например, после предварительной фильтрации)
+ ///
private HashSet _activeNodes;
+ ///
+ /// Список точек с которых начинается поиск в графе для расширения
+ ///
private HashSet _entryNodes;
+ ///
+ /// Режим работы алгоритма расширения, зависящий от того заданы ли ограничения в точках, и заданы ли точки начала поиска
+ ///
private Mode _mode;
public Mode NodeCheckMode => _mode;
@@ -28,11 +37,14 @@ namespace ZeroLevel.HNSW
_mode = Mode.None;
}
+ ///
+ /// Расчет процентного содержания точек доступных для использования в данном контексте, по отношению к общему количеству точек
+ ///
public SearchContext CaclulatePercentage(long total)
{
- if (total > 0)
+ if ((_mode == Mode.ActiveCheck || _mode == Mode.ActiveInactiveCheck) && total > 0)
{
- PercentInTotal = ((_activeNodes.Count * 100d) / (double)total) / 100.0d;
+ PercentInTotal = ((_activeNodes?.Count ?? 0 * 100d) / (double)total) / 100.0d;
}
return this;
}
@@ -43,14 +55,26 @@ namespace ZeroLevel.HNSW
return this;
}
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private bool _isActiveNode(int nodeId) => _activeNodes?.Contains(nodeId) ?? false;
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private bool _isEntryNode(int nodeId) => _entryNodes?.Contains(nodeId) ?? false;
+
+
+ ///
+ /// Проверка, подходит ли указанная точка для включения в набор расширения
+ ///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal bool IsActiveNode(int nodeId)
{
switch (_mode)
{
- case Mode.ActiveCheck: return _activeNodes.Contains(nodeId);
- case Mode.InactiveCheck: return _entryNodes.Contains(nodeId) == false;
- case Mode.ActiveInactiveCheck: return _entryNodes.Contains(nodeId) == false && _activeNodes.Contains(nodeId);
+ // Если задан набор разрешенных к использованию точек, проверяется вхождение в него
+ case Mode.ActiveCheck: return _isActiveNode(nodeId);
+ // Если задан набор точек начала поиска, проверка невхождения точки в него
+ case Mode.InactiveCheck: return _isEntryNode(nodeId) == false;
+ // Если задан и ограничивающий и начальный наборы точек, проверка и на ограничение и на невхождение в начальный набор
+ case Mode.ActiveInactiveCheck: return false == _isEntryNode(nodeId) && _isActiveNode(nodeId);
}
return nodeId >= 0;
}
diff --git a/ZeroLevel.Qdrant/DoubleExtensions.cs b/ZeroLevel.Qdrant/DoubleExtensions.cs
deleted file mode 100644
index 1062815..0000000
--- a/ZeroLevel.Qdrant/DoubleExtensions.cs
+++ /dev/null
@@ -1,10 +0,0 @@
-namespace ZeroLevel.Qdrant
-{
- public static class DoubleExtensions
- {
- public static string ConvertToString(this double num)
- {
- return num.ToString().Replace(',', '.');
- }
- }
-}
diff --git a/ZeroLevel.Qdrant/FloatExtensions.cs b/ZeroLevel.Qdrant/FloatExtensions.cs
new file mode 100644
index 0000000..891871e
--- /dev/null
+++ b/ZeroLevel.Qdrant/FloatExtensions.cs
@@ -0,0 +1,18 @@
+using System.Globalization;
+
+namespace ZeroLevel.Qdrant
+{
+ public static class NumericExtensions
+ {
+ private static NumberFormatInfo nfi = new NumberFormatInfo() { NumberDecimalSeparator = "." };
+ public static string ConvertToString(this float num)
+ {
+ return num.ToString(nfi);
+ }
+
+ public static string ConvertToString(this double num)
+ {
+ return num.ToString(nfi);
+ }
+ }
+}
diff --git a/ZeroLevel.Qdrant/Models/Point.cs b/ZeroLevel.Qdrant/Models/Point.cs
index 64f0121..b992d45 100644
--- a/ZeroLevel.Qdrant/Models/Point.cs
+++ b/ZeroLevel.Qdrant/Models/Point.cs
@@ -4,6 +4,6 @@
{
public long id { get; set; }
public dynamic payload;
- public double[] vector;
+ public float[] vector;
}
}
diff --git a/ZeroLevel.Qdrant/Models/Requests/CreateCollectionReqeust.cs b/ZeroLevel.Qdrant/Models/Requests/CreateCollectionReqeust.cs
index 19c6cec..5c177a3 100644
--- a/ZeroLevel.Qdrant/Models/Requests/CreateCollectionReqeust.cs
+++ b/ZeroLevel.Qdrant/Models/Requests/CreateCollectionReqeust.cs
@@ -1,27 +1,17 @@
namespace ZeroLevel.Qdrant.Models.Requests
{
- internal sealed class CreateCollectionParameters
+ internal sealed class CreateCollectionReqeust
{
- public string name { get; set; }
public string distance { get; set; }
public int vector_size { get; set; }
-
public bool? on_disk_payload { get; set; }
- }
- internal sealed class CreateCollectionReqeust
- {
- public CreateCollectionParameters create_collection { get; set; }
- public CreateCollectionReqeust(string name, string distance, int vector_size,
+ public CreateCollectionReqeust(string distance, int vector_size,
bool? on_disk_payload = null)
{
- create_collection = new CreateCollectionParameters
- {
- name = name,
- distance = distance,
- vector_size = vector_size,
- on_disk_payload = on_disk_payload
- };
+ this.distance = distance;
+ this.vector_size = vector_size;
+ this.on_disk_payload = on_disk_payload;
}
}
}
diff --git a/ZeroLevel.Qdrant/Models/Requests/CreateIndexRequest.cs b/ZeroLevel.Qdrant/Models/Requests/CreateIndexRequest.cs
index 663dc63..48c7eb7 100644
--- a/ZeroLevel.Qdrant/Models/Requests/CreateIndexRequest.cs
+++ b/ZeroLevel.Qdrant/Models/Requests/CreateIndexRequest.cs
@@ -1,8 +1,34 @@
namespace ZeroLevel.Qdrant.Models.Requests
{
+ public enum IndexFieldType
+ {
+ Keyword,
+ Integer,
+ Float,
+ Geo
+ }
+
+ ///
+ /// Available field types are:
+ /// keyword - for keyword payload, affects Match filtering conditions.
+ /// integer - for integer payload, affects Match and Range filtering conditions.
+ /// float - for float payload, affects Range filtering conditions.
+ /// geo - for geo payload, affects Geo Bounding Box and Geo Radius filtering conditions.
+ ///
internal sealed class CreateIndexRequest
{
- public CreateIndexRequest(string name) => create_index = name;
- public string create_index { get; set; }
+ public string field_name { get; set; }
+ public string field_type { get; set; }
+ public CreateIndexRequest(string name, IndexFieldType type)
+ {
+ field_name = name;
+ switch (type)
+ {
+ case IndexFieldType.Integer: field_type = "integer"; break;
+ case IndexFieldType.Float: field_type = "float"; break;
+ case IndexFieldType.Geo: field_type = "geo"; break;
+ case IndexFieldType.Keyword: field_type = "keyword"; break;
+ }
+ }
}
}
diff --git a/ZeroLevel.Qdrant/Models/Requests/DeleteCollectionRequest.cs b/ZeroLevel.Qdrant/Models/Requests/DeleteCollectionRequest.cs
deleted file mode 100644
index 4cfe690..0000000
--- a/ZeroLevel.Qdrant/Models/Requests/DeleteCollectionRequest.cs
+++ /dev/null
@@ -1,8 +0,0 @@
-namespace ZeroLevel.Qdrant.Models.Responces
-{
- internal sealed class DeleteCollectionRequest
- {
- public DeleteCollectionRequest(string name) => delete_collection = name;
- public string delete_collection { get; set; }
- }
-}
diff --git a/ZeroLevel.Qdrant/Models/Requests/PointsRequest.cs b/ZeroLevel.Qdrant/Models/Requests/PointsRequest.cs
index 9fe7799..071656d 100644
--- a/ZeroLevel.Qdrant/Models/Requests/PointsRequest.cs
+++ b/ZeroLevel.Qdrant/Models/Requests/PointsRequest.cs
@@ -3,7 +3,7 @@
internal sealed class PointsRequest
{
public long[] ids { get; set; }
- /*public bool with_payload { get; set; } = true;
- public bool with_vector { get; set; } = true;*/
+ public bool with_payload { get; set; } = true;
+ public bool with_vector { get; set; } = false;
}
}
diff --git a/ZeroLevel.Qdrant/Models/Requests/PointsUploadRequest.cs b/ZeroLevel.Qdrant/Models/Requests/PointsUploadRequest.cs
deleted file mode 100644
index 9c2cf7f..0000000
--- a/ZeroLevel.Qdrant/Models/Requests/PointsUploadRequest.cs
+++ /dev/null
@@ -1,101 +0,0 @@
-using System;
-using System.Linq;
-using System.Text;
-using ZeroLevel.Qdrant.Services;
-using ZeroLevel.Services.Collections;
-
-namespace ZeroLevel.Qdrant.Models.Requests
-{
- /*
- integer - 64-bit integer in the range -9223372036854775808 to 9223372036854775807. array of long
- float - 64-bit floating point number. array of double
- keyword - string value. array of strings
- geo - Geographical coordinates. Example: { "lon": 52.5200, "lat": 13.4050 } array of lon&lat of double
- */
-
- public sealed class UpsertPoint
- {
- public long? id { get; set; } = null;
- public T payload { get; set; }
- public double[] vector { get; set; }
- }
- public sealed class UpsertPoints
- {
- public UpsertPoint[] points { get; set; }
- }
- public sealed class PointsUploadRequest
- {
- private static IEverythingStorage _cachee = EverythingStorage.Create();
- public UpsertPoints upsert_points { get; set; }
-
- public string ToJSON()
- {
- if (!_cachee.ContainsKey>("converter"))
- {
- _cachee.Add("converter", new QdrantJsonConverter());
- }
- var converter = _cachee.Get>("converter");
-
- Func, string> p_conv = up =>
- {
- if (up.id != null)
- {
- return $"\"id\": {up.id}, \"payload\": {{ {converter.ToJson(up.payload)} }}, \"vector\": [{ string.Join(",", up.vector.Select(f => f.ConvertToString()))}]";
- }
- return $"\"payload\": {{ {converter.ToJson(up.payload)} }}, \"vector\": [{ string.Join(",", up.vector.Select(f => f.ConvertToString()))}]";
- };
-
- var json = new StringBuilder();
- json.Append("{");
- json.Append("\"upsert_points\": {");
- json.Append("\"points\":[ {");
- json.Append(string.Join("},{", upsert_points.points.Select(p => p_conv(p))));
- json.Append("}]");
- json.Append("}");
- json.Append("}");
- return json.ToString();
- }
- }
-
- public sealed class ColumnPoints
- {
- public long[] ids { get; set; }
- public T[] payloads { get; set; }
- public double[,] vectors { get; set; }
- }
-
- public sealed class UpsertColumnPoints
- {
- public ColumnPoints batch { get; set; }
- }
-
- public sealed class PointsColumnUploadRequest
- {
- private static IEverythingStorage _cachee = EverythingStorage.Create();
- public UpsertColumnPoints upsert_points { get; set; }
-
- public string ToJSON()
- {
- if (!_cachee.ContainsKey>("converter"))
- {
- _cachee.Add("converter", new QdrantJsonConverter());
- }
- var converter = _cachee.Get>("converter");
-
- var json = new StringBuilder();
- json.Append("{");
- json.Append("\"upsert_points\": {");
- json.Append("\"batch\": {");
- if (upsert_points.batch.ids != null && upsert_points.batch.ids.Length > 0)
- {
- json.Append($"\"ids\": [{string.Join(",", upsert_points.batch.ids)}], ");
- }
- json.Append($"\"payloads\": [ {{ {string.Join("} ,{ ", upsert_points.batch.payloads.Select(payload => converter.ToJson(payload)))} }} ], ");
- json.Append($"\"vectors\": [{string.Join(",", Enumerable.Range(0, upsert_points.batch.vectors.GetLength(0)).Select(row => "[" + string.Join(",", ArrayExtensions.GetRow(upsert_points.batch.vectors, row).Select(f => f.ConvertToString())) + "]"))}]");
- json.Append("}");
- json.Append("}");
- json.Append("}");
- return json.ToString();
- }
- }
-}
diff --git a/ZeroLevel.Qdrant/Models/Requests/PointsUpsertRequest.cs b/ZeroLevel.Qdrant/Models/Requests/PointsUpsertRequest.cs
new file mode 100644
index 0000000..41a2311
--- /dev/null
+++ b/ZeroLevel.Qdrant/Models/Requests/PointsUpsertRequest.cs
@@ -0,0 +1,46 @@
+using System;
+using System.Linq;
+using System.Text;
+using ZeroLevel.Qdrant.Services;
+
+namespace ZeroLevel.Qdrant.Models.Requests
+{
+ /*
+ integer - 64-bit integer in the range -9223372036854775808 to 9223372036854775807. array of long
+ float - 64-bit floating point number. array of double
+ keyword - string value. array of strings
+ geo - Geographical coordinates. Example: { "lon": 52.5200, "lat": 13.4050 } array of lon&lat of double
+ */
+
+ public sealed class UpsertPoint
+ {
+ public long? id { get; set; } = null;
+ public T payload { get; set; }
+ public float[] vector { get; set; }
+ }
+
+ public sealed class PointsUpsertRequest
+ {
+ public UpsertPoint[] points { get; set; }
+
+ public string ToJSON()
+ {
+ if (points != null && points.Length > 0)
+ {
+ var dims = points[0].vector.Length;
+ Func converter = o => QdrantJsonConverter.ConvertToJson(o);
+ var json = new StringBuilder();
+ json.Append("{");
+ json.Append("\"batch\": {");
+ json.Append($"\"ids\": [{string.Join(",", points.Select(p => p.id))}], ");
+ json.Append($"\"payloads\": [ {{ {string.Join("} ,{ ", points.Select(p => converter(p.payload)))} }} ], ");
+ json.Append($"\"vectors\": [{string.Join(", ", points.Select(p => QdrantJsonConverter.ConvertToJson(p.vector)))}]");
+ json.Append("}");
+ json.Append("}");
+ return json.ToString();
+ }
+ return String.Empty;
+ }
+
+ }
+}
diff --git a/ZeroLevel.Qdrant/Models/Responces/PointResponse.cs b/ZeroLevel.Qdrant/Models/Responces/PointResponse.cs
index 54c92b1..114532c 100644
--- a/ZeroLevel.Qdrant/Models/Responces/PointResponse.cs
+++ b/ZeroLevel.Qdrant/Models/Responces/PointResponse.cs
@@ -1,9 +1,16 @@
namespace ZeroLevel.Qdrant.Models.Responces
{
- public sealed class PointResponse
+ public sealed class PointsResponse
{
public Point[] result { get; set; }
public string status { get; set; }
public float time { get; set; }
}
+
+ public sealed class PointResponse
+ {
+ public Point result { get; set; }
+ public string status { get; set; }
+ public float time { get; set; }
+ }
}
diff --git a/ZeroLevel.Qdrant/Models/Responces/SearchResponse.cs b/ZeroLevel.Qdrant/Models/Responces/SearchResponse.cs
index efcb437..2d173aa 100644
--- a/ZeroLevel.Qdrant/Models/Responces/SearchResponse.cs
+++ b/ZeroLevel.Qdrant/Models/Responces/SearchResponse.cs
@@ -3,7 +3,7 @@
public sealed class ScoredPoint
{
public long id { get; set; }
- public double score { get; set; }
+ public float score { get; set; }
}
public sealed class SearchResponse
{
diff --git a/ZeroLevel.Qdrant/QdrantClient.cs b/ZeroLevel.Qdrant/QdrantClient.cs
index 77edad8..0f38890 100644
--- a/ZeroLevel.Qdrant/QdrantClient.cs
+++ b/ZeroLevel.Qdrant/QdrantClient.cs
@@ -20,6 +20,7 @@ namespace ZeroLevel.Qdrant
///
public class QdrantClient
{
+ private const int DEFAULT_OPERATION_TIMEOUT_S = 30;
private HttpClient CreateClient()
{
var handler = new HttpClientHandler
@@ -41,7 +42,7 @@ namespace ZeroLevel.Qdrant
#region API
- #region Collection https://qdrant.tech/documentation/collections/
+ #region Collection https://qdrant.github.io/qdrant/redoc/index.html#tag/collections
///
/// Create collection
///
@@ -53,12 +54,12 @@ namespace ZeroLevel.Qdrant
{
try
{
- var collection = new CreateCollectionReqeust(name, distance, vector_size, on_disk_payload);
+ var collection = new CreateCollectionReqeust(distance, vector_size, on_disk_payload);
var json = JsonConvert.SerializeObject(collection);
var data = new StringContent(json, Encoding.UTF8, "application/json");
- var url = $"/collections";
+ var url = $"/collections/{name}";
- var response = await _request(url, new HttpMethod("POST"), data);
+ var response = await _request(url, new HttpMethod("PUT"), data);
return InvokeResult.Succeeding(response);
}
catch (Exception ex)
@@ -71,16 +72,12 @@ namespace ZeroLevel.Qdrant
/// Delete collection by name
///
/// Collection name
- public async Task> DeleteCollection(string name)
+ public async Task> DeleteCollection(string name, int timeout = DEFAULT_OPERATION_TIMEOUT_S)
{
try
{
- var collection = new DeleteCollectionRequest(name);
- var json = JsonConvert.SerializeObject(collection);
- var data = new StringContent(json, Encoding.UTF8, "application/json");
- var url = $"/collections";
-
- var response = await _request(url, new HttpMethod("POST"), data);
+ var url = $"/collections/{name}?timeout={timeout}";
+ var response = await _request(url, new HttpMethod("DELETE"), null);
return InvokeResult.Succeeding(response);
}
catch (Exception ex)
@@ -95,16 +92,16 @@ namespace ZeroLevel.Qdrant
///
/// For indexing, it is recommended to choose the field that limits the search result the most. As a rule, the more different values a payload value has, the more efficient the index will be used. You should not create an index for Boolean fields and fields with only a few possible values.
///
- public async Task> CreateIndex(string collection_name, string field_name)
+ public async Task> CreateIndex(string collection_name, string field_name, IndexFieldType field_type)
{
try
{
- var index = new CreateIndexRequest(field_name);
+ var index = new CreateIndexRequest(field_name, field_type);
var json = JsonConvert.SerializeObject(index);
var data = new StringContent(json, Encoding.UTF8, "application/json");
- var url = $"/collections/{collection_name}";
+ var url = $"/collections/{collection_name}/index";
- var response = await _request(url, new HttpMethod("POST"), data);
+ var response = await _request(url, new HttpMethod("PUT"), data);
return InvokeResult.Succeeding(response);
}
catch (Exception ex)
@@ -164,7 +161,25 @@ namespace ZeroLevel.Qdrant
///
/// There is a method for retrieving points by their ids.
///
- public async Task> Points(string collection_name, long[] ids)
+ public async Task> GetPoint(string collection_name, long id)
+ {
+ try
+ {
+ string url = $"/collections/{collection_name}/points/{id}";
+ var response = await _request(url, new HttpMethod("GET"), null);
+ return InvokeResult.Succeeding(response);
+ }
+ catch (Exception ex)
+ {
+ Log.Error(ex, $"[QdrantClient.Points] Collection name: {collection_name}.");
+ return InvokeResult.Fault($"[QdrantClient.GetPoint] Collection name: {collection_name}. Point ID: {id}\r\n{ex.ToString()}");
+ }
+ }
+
+ ///
+ /// There is a method for retrieving points by their ids.
+ ///
+ public async Task> GetPoints(string collection_name, long[] ids)
{
try
{
@@ -172,13 +187,13 @@ namespace ZeroLevel.Qdrant
var json = JsonConvert.SerializeObject(points);
var data = new StringContent(json, Encoding.UTF8, "application/json");
string url = $"/collections/{collection_name}/points";
- var response = await _request(url, new HttpMethod("POST"), data);
- return InvokeResult.Succeeding(response);
+ var response = await _request(url, new HttpMethod("POST"), data);
+ return InvokeResult.Succeeding(response);
}
catch (Exception ex)
{
Log.Error(ex, $"[QdrantClient.Points] Collection name: {collection_name}.");
- return InvokeResult.Fault($"[QdrantClient.Points] Collection name: {collection_name}.\r\n{ex.ToString()}");
+ return InvokeResult.Fault($"[QdrantClient.GetPoints] Collection name: {collection_name}.\r\n{ex.ToString()}");
}
}
@@ -208,43 +223,20 @@ namespace ZeroLevel.Qdrant
///
/// Record-oriented of creating batches
///
- public async Task> PointsUpload(string collection_name, UpsertPoint[] points)
+ public async Task> UpsertPoints(string collection_name, PointsUpsertRequest points)
{
try
{
- var points_request = new PointsUploadRequest { upsert_points = new UpsertPoints { points = points } };
- var json = points_request.ToJSON();
+ var json = points.ToJSON();
var data = new StringContent(json, Encoding.UTF8, "application/json");
- var url = $"/collections/{collection_name}";
-
- var response = await _request(url, new HttpMethod("POST"), data);
+ var url = $"/collections/{collection_name}/points";
+ var response = await _request(url, new HttpMethod("PUT"), data);
return InvokeResult.Succeeding(response);
}
catch (Exception ex)
{
- Log.Error(ex, $"[QdrantClient.Points] Collection name: {collection_name}.");
- return InvokeResult.Fault($"[QdrantClient.Points] Collection name: {collection_name}.\r\n{ex.ToString()}");
- }
- }
- ///
- /// Column-oriented of creating batches
- ///
- public async Task> PointsColumnUpload(string collection_name, ColumnPoints points)
- {
- try
- {
- var points_request = new PointsColumnUploadRequest { upsert_points = new UpsertColumnPoints { batch = points } };
- var json = points_request.ToJSON();
- var data = new StringContent(json, Encoding.UTF8, "application/json");
- var url = $"/collections/{collection_name}";
-
- var response = await _request(url, new HttpMethod("POST"), data);
- return InvokeResult.Succeeding(response);
- }
- catch (Exception ex)
- {
- Log.Error(ex, $"[QdrantClient.Points] Collection name: {collection_name}.");
- return InvokeResult.Fault($"[QdrantClient.Points] Collection name: {collection_name}.\r\n{ex.ToString()}");
+ Log.Error(ex, $"[QdrantClient.UpsertPoints] Collection name: {collection_name}.");
+ return InvokeResult.Fault($"[QdrantClient.UpsertPoints] Collection name: {collection_name}.\r\n{ex.ToString()}");
}
}
///
diff --git a/ZeroLevel.Qdrant/Services/QdrantJsonConverter.cs b/ZeroLevel.Qdrant/Services/QdrantJsonConverter.cs
index 69aec5f..cf2cb56 100644
--- a/ZeroLevel.Qdrant/Services/QdrantJsonConverter.cs
+++ b/ZeroLevel.Qdrant/Services/QdrantJsonConverter.cs
@@ -11,7 +11,7 @@ using ZeroLevel.Services.Serialization;
namespace ZeroLevel.Qdrant.Services
{
- public class QdrantJsonConverter
+ public static class QdrantJsonConverter
{
private static string KeywordToString(IMemberInfo member, object v)
{
@@ -36,7 +36,12 @@ namespace ZeroLevel.Qdrant.Services
private const string GEO_TYPE = "geo";
private const string FLOAT_TYPE = "float";
private const string INTEGER_TYPE = "integer";
- public string ToJson(T value)
+
+ public static string ConvertToJson(float[] vector)
+ {
+ return "[" + string.Join(", ", vector.Select(f => f.ConvertToString())) + "]";
+ }
+ public static string ConvertToJson(T value)
{
var json = new StringBuilder();
@@ -154,7 +159,7 @@ namespace ZeroLevel.Qdrant.Services
return json.ToString();
}
- public IEnumerable