ogoun
/
Zero
mirror of https://github.com/ogoun/Zero.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update

Browse Source
pull/4/head
Ogoun 2 years ago
parent 3e9b6de12e
commit 5cf967fc61
15 changed files with 190 additions and 197 deletions
Show Stats Download Patch File Download Diff File

34
ZeroLevel.HNSW/Model/SearchContext.cs
Unescape View File

@ -15,8 +15,17 @@ namespace ZeroLevel.HNSW
public sealed class SearchContext
{
/// <summary>
/// Список номеров которые разрешены к добавлению итогового результата, если поиск ведется в ограниченном наборе точек (например, после предварительной фильтрации)
/// </summary>
private HashSet<int> _activeNodes;
/// <summary>
/// Список точек с которых начинается поиск в графе для расширения
/// </summary>
private HashSet<int> _entryNodes;
/// <summary>
/// Режим работы алгоритма расширения, зависящий от того заданы ли ограничения в точках, и заданы ли точки начала поиска
/// </summary>
private Mode _mode;
public Mode NodeCheckMode => _mode;
@ -28,11 +37,14 @@ namespace ZeroLevel.HNSW
_mode = Mode.None;
}
/// <summary>
/// Расчет процентного содержания точек доступных для использования в данном контексте, по отношению к общему количеству точек
/// </summary>
public SearchContext CaclulatePercentage(long total)
{
if (total > 0)
if ((_mode == Mode.ActiveCheck || _mode == Mode.ActiveInactiveCheck) && total > 0)
{
PercentInTotal = ((_activeNodes.Count * 100d) / (double)total) / 100.0d;
PercentInTotal = ((_activeNodes?.Count ?? 0 * 100d) / (double)total) / 100.0d;
}
return this;
}
@ -43,14 +55,26 @@ namespace ZeroLevel.HNSW
return this;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool _isActiveNode(int nodeId) => _activeNodes?.Contains(nodeId) ?? false;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool _isEntryNode(int nodeId) => _entryNodes?.Contains(nodeId) ?? false;
/// <summary>
/// Проверка, подходит ли указанная точка для включения в набор расширения
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal bool IsActiveNode(int nodeId)
{
switch (_mode)
{
case Mode.ActiveCheck: return _activeNodes.Contains(nodeId);
case Mode.InactiveCheck: return _entryNodes.Contains(nodeId) == false;
case Mode.ActiveInactiveCheck: return _entryNodes.Contains(nodeId) == false && _activeNodes.Contains(nodeId);
// Если задан набор разрешенных к использованию точек, проверяется вхождение в него
case Mode.ActiveCheck: return _isActiveNode(nodeId);
// Если задан набор точек начала поиска, проверка невхождения точки в него
case Mode.InactiveCheck: return _isEntryNode(nodeId) == false;
// Если задан и ограничивающий и начальный наборы точек, проверка и на ограничение и на невхождение в начальный набор
case Mode.ActiveInactiveCheck: return false == _isEntryNode(nodeId) && _isActiveNode(nodeId);
}
return nodeId >= 0;
}

10
ZeroLevel.Qdrant/DoubleExtensions.cs
Unescape View File

@ -1,10 +0,0 @@
namespace ZeroLevel.Qdrant
{
public static class DoubleExtensions
{
public static string ConvertToString(this double num)
{
return num.ToString().Replace(',', '.');
}
}
}

18
ZeroLevel.Qdrant/FloatExtensions.cs
Unescape View File

@ -0,0 +1,18 @@
using System.Globalization;
namespace ZeroLevel.Qdrant
{
public static class NumericExtensions
{
private static NumberFormatInfo nfi = new NumberFormatInfo() { NumberDecimalSeparator = "." };
public static string ConvertToString(this float num)
{
return num.ToString(nfi);
}
public static string ConvertToString(this double num)
{
return num.ToString(nfi);
}
}
}

2
ZeroLevel.Qdrant/Models/Point.cs
Unescape View File

@ -4,6 +4,6 @@
{
public long id { get; set; }
public dynamic payload;
public double[] vector;
public float[] vector;
}
}

20
ZeroLevel.Qdrant/Models/Requests/CreateCollectionReqeust.cs
Unescape View File

@ -1,27 +1,17 @@
namespace ZeroLevel.Qdrant.Models.Requests
{
internal sealed class CreateCollectionParameters
internal sealed class CreateCollectionReqeust
{
public string name { get; set; }
public string distance { get; set; }
public int vector_size { get; set; }
public bool? on_disk_payload { get; set; }
}
internal sealed class CreateCollectionReqeust
{
public CreateCollectionParameters create_collection { get; set; }
public CreateCollectionReqeust(string name, string distance, int vector_size,
public CreateCollectionReqeust(string distance, int vector_size,
bool? on_disk_payload = null)
{
create_collection = new CreateCollectionParameters
{
name = name,
distance = distance,
vector_size = vector_size,
on_disk_payload = on_disk_payload
};
this.distance = distance;
this.vector_size = vector_size;
this.on_disk_payload = on_disk_payload;
}
}
}

30
ZeroLevel.Qdrant/Models/Requests/CreateIndexRequest.cs
Unescape View File

@ -1,8 +1,34 @@
namespace ZeroLevel.Qdrant.Models.Requests
{
public enum IndexFieldType
{
Keyword,
Integer,
Float,
Geo
}
/// <summary>
/// Available field types are:
/// keyword - for keyword payload, affects Match filtering conditions.
/// integer - for integer payload, affects Match and Range filtering conditions.
/// float - for float payload, affects Range filtering conditions.
/// geo - for geo payload, affects Geo Bounding Box and Geo Radius filtering conditions.
/// </summary>
internal sealed class CreateIndexRequest
{
public CreateIndexRequest(string name) => create_index = name;
public string create_index { get; set; }
public string field_name { get; set; }
public string field_type { get; set; }
public CreateIndexRequest(string name, IndexFieldType type)
{
field_name = name;
switch (type)
{
case IndexFieldType.Integer: field_type = "integer"; break;
case IndexFieldType.Float: field_type = "float"; break;
case IndexFieldType.Geo: field_type = "geo"; break;
case IndexFieldType.Keyword: field_type = "keyword"; break;
}
}
}
}

8
ZeroLevel.Qdrant/Models/Requests/DeleteCollectionRequest.cs
Unescape View File

@ -1,8 +0,0 @@
namespace ZeroLevel.Qdrant.Models.Responces
{
internal sealed class DeleteCollectionRequest
{
public DeleteCollectionRequest(string name) => delete_collection = name;
public string delete_collection { get; set; }
}
}

4
ZeroLevel.Qdrant/Models/Requests/PointsRequest.cs
Unescape View File

@ -3,7 +3,7 @@
internal sealed class PointsRequest
{
public long[] ids { get; set; }
/*public bool with_payload { get; set; } = true;
public bool with_vector { get; set; } = true;*/
public bool with_payload { get; set; } = true;
public bool with_vector { get; set; } = false;
}
}

101
ZeroLevel.Qdrant/Models/Requests/PointsUploadRequest.cs
Unescape View File

@ -1,101 +0,0 @@
using System;
using System.Linq;
using System.Text;
using ZeroLevel.Qdrant.Services;
using ZeroLevel.Services.Collections;
namespace ZeroLevel.Qdrant.Models.Requests
{
/*
integer - 64-bit integer in the range -9223372036854775808 to 9223372036854775807. array of long
float - 64-bit floating point number. array of double
keyword - string value. array of strings
geo - Geographical coordinates. Example: { "lon": 52.5200, "lat": 13.4050 } array of lon&lat of double
*/
public sealed class UpsertPoint<T>
{
public long? id { get; set; } = null;
public T payload { get; set; }
public double[] vector { get; set; }
}
public sealed class UpsertPoints<T>
{
public UpsertPoint<T>[] points { get; set; }
}
public sealed class PointsUploadRequest<T>
{
private static IEverythingStorage _cachee = EverythingStorage.Create();
public UpsertPoints<T> upsert_points { get; set; }
public string ToJSON()
{
if (!_cachee.ContainsKey<QdrantJsonConverter<T>>("converter"))
{
_cachee.Add("converter", new QdrantJsonConverter<T>());
}
var converter = _cachee.Get<QdrantJsonConverter<T>>("converter");
Func<UpsertPoint<T>, string> p_conv = up =>
{
if (up.id != null)
{
return $"\"id\": {up.id}, \"payload\": {{ {converter.ToJson(up.payload)} }}, \"vector\": [{ string.Join(",", up.vector.Select(f => f.ConvertToString()))}]";
}
return $"\"payload\": {{ {converter.ToJson(up.payload)} }}, \"vector\": [{ string.Join(",", up.vector.Select(f => f.ConvertToString()))}]";
};
var json = new StringBuilder();
json.Append("{");
json.Append("\"upsert_points\": {");
json.Append("\"points\":[ {");
json.Append(string.Join("},{", upsert_points.points.Select(p => p_conv(p))));
json.Append("}]");
json.Append("}");
json.Append("}");
return json.ToString();
}
}
public sealed class ColumnPoints<T>
{
public long[] ids { get; set; }
public T[] payloads { get; set; }
public double[,] vectors { get; set; }
}
public sealed class UpsertColumnPoints<T>
{
public ColumnPoints<T> batch { get; set; }
}
public sealed class PointsColumnUploadRequest<T>
{
private static IEverythingStorage _cachee = EverythingStorage.Create();
public UpsertColumnPoints<T> upsert_points { get; set; }
public string ToJSON()
{
if (!_cachee.ContainsKey<QdrantJsonConverter<T>>("converter"))
{
_cachee.Add("converter", new QdrantJsonConverter<T>());
}
var converter = _cachee.Get<QdrantJsonConverter<T>>("converter");
var json = new StringBuilder();
json.Append("{");
json.Append("\"upsert_points\": {");
json.Append("\"batch\": {");
if (upsert_points.batch.ids != null && upsert_points.batch.ids.Length > 0)
{
json.Append($"\"ids\": [{string.Join(",", upsert_points.batch.ids)}], ");
}
json.Append($"\"payloads\": [ {{ {string.Join("} ,{ ", upsert_points.batch.payloads.Select(payload => converter.ToJson(payload)))} }} ], ");
json.Append($"\"vectors\": [{string.Join(",", Enumerable.Range(0, upsert_points.batch.vectors.GetLength(0)).Select(row => "[" + string.Join(",", ArrayExtensions.GetRow(upsert_points.batch.vectors, row).Select(f => f.ConvertToString())) + "]"))}]");
json.Append("}");
json.Append("}");
json.Append("}");
return json.ToString();
}
}
}

46
ZeroLevel.Qdrant/Models/Requests/PointsUpsertRequest.cs
Unescape View File

@ -0,0 +1,46 @@
using System;
using System.Linq;
using System.Text;
using ZeroLevel.Qdrant.Services;
namespace ZeroLevel.Qdrant.Models.Requests
{
/*
integer - 64-bit integer in the range -9223372036854775808 to 9223372036854775807. array of long
float - 64-bit floating point number. array of double
keyword - string value. array of strings
geo - Geographical coordinates. Example: { "lon": 52.5200, "lat": 13.4050 } array of lon&lat of double
*/
public sealed class UpsertPoint<T>
{
public long? id { get; set; } = null;
public T payload { get; set; }
public float[] vector { get; set; }
}
public sealed class PointsUpsertRequest<T>
{
public UpsertPoint<T>[] points { get; set; }
public string ToJSON()
{
if (points != null && points.Length > 0)
{
var dims = points[0].vector.Length;
Func<T, string> converter = o => QdrantJsonConverter<T>.ConvertToJson(o);
var json = new StringBuilder();
json.Append("{");
json.Append("\"batch\": {");
json.Append($"\"ids\": [{string.Join(",", points.Select(p => p.id))}], ");
json.Append($"\"payloads\": [ {{ {string.Join("} ,{ ", points.Select(p => converter(p.payload)))} }} ], ");
json.Append($"\"vectors\": [{string.Join(", ", points.Select(p => QdrantJsonConverter<T>.ConvertToJson(p.vector)))}]");
json.Append("}");
json.Append("}");
return json.ToString();
}
return String.Empty;
}
}
}

9
ZeroLevel.Qdrant/Models/Responces/PointResponse.cs
Unescape View File

@ -1,9 +1,16 @@
namespace ZeroLevel.Qdrant.Models.Responces
{
public sealed class PointResponse
public sealed class PointsResponse
{
public Point[] result { get; set; }
public string status { get; set; }
public float time { get; set; }
}
public sealed class PointResponse
{
public Point result { get; set; }
public string status { get; set; }
public float time { get; set; }
}
}

2
ZeroLevel.Qdrant/Models/Responces/SearchResponse.cs
Unescape View File

@ -3,7 +3,7 @@
public sealed class ScoredPoint
{
public long id { get; set; }
public double score { get; set; }
public float score { get; set; }
}
public sealed class SearchResponse
{

88
ZeroLevel.Qdrant/QdrantClient.cs
Unescape View File

@ -20,6 +20,7 @@ namespace ZeroLevel.Qdrant
/// </summary>
public class QdrantClient
{
private const int DEFAULT_OPERATION_TIMEOUT_S = 30;
private HttpClient CreateClient()
{
var handler = new HttpClientHandler
@ -41,7 +42,7 @@ namespace ZeroLevel.Qdrant
#region API
#region Collection https://qdrant.tech/documentation/collections/
#region Collection https://qdrant.github.io/qdrant/redoc/index.html#tag/collections
/// <summary>
/// Create collection
/// </summary>
@ -53,12 +54,12 @@ namespace ZeroLevel.Qdrant
{
try
{
var collection = new CreateCollectionReqeust(name, distance, vector_size, on_disk_payload);
var collection = new CreateCollectionReqeust(distance, vector_size, on_disk_payload);
var json = JsonConvert.SerializeObject(collection);
var data = new StringContent(json, Encoding.UTF8, "application/json");
var url = $"/collections";
var url = $"/collections/{name}";
var response = await _request<OperationResponse>(url, new HttpMethod("POST"), data);
var response = await _request<OperationResponse>(url, new HttpMethod("PUT"), data);
return InvokeResult.Succeeding<OperationResponse>(response);
}
catch (Exception ex)
@ -71,16 +72,12 @@ namespace ZeroLevel.Qdrant
/// Delete collection by name
/// </summary>
/// <param name="name">Collection name</param>
public async Task<InvokeResult<OperationResponse>> DeleteCollection(string name)
public async Task<InvokeResult<OperationResponse>> DeleteCollection(string name, int timeout = DEFAULT_OPERATION_TIMEOUT_S)
{
try
{
var collection = new DeleteCollectionRequest(name);
var json = JsonConvert.SerializeObject(collection);
var data = new StringContent(json, Encoding.UTF8, "application/json");
var url = $"/collections";
var response = await _request<OperationResponse>(url, new HttpMethod("POST"), data);
var url = $"/collections/{name}?timeout={timeout}";
var response = await _request<OperationResponse>(url, new HttpMethod("DELETE"), null);
return InvokeResult.Succeeding<OperationResponse>(response);
}
catch (Exception ex)
@ -95,16 +92,16 @@ namespace ZeroLevel.Qdrant
/// <summary>
/// For indexing, it is recommended to choose the field that limits the search result the most. As a rule, the more different values a payload value has, the more efficient the index will be used. You should not create an index for Boolean fields and fields with only a few possible values.
/// </summary>
public async Task<InvokeResult<CreateIndexResponse>> CreateIndex(string collection_name, string field_name)
public async Task<InvokeResult<CreateIndexResponse>> CreateIndex(string collection_name, string field_name, IndexFieldType field_type)
{
try
{
var index = new CreateIndexRequest(field_name);
var index = new CreateIndexRequest(field_name, field_type);
var json = JsonConvert.SerializeObject(index);
var data = new StringContent(json, Encoding.UTF8, "application/json");
var url = $"/collections/{collection_name}";
var url = $"/collections/{collection_name}/index";
var response = await _request<CreateIndexResponse>(url, new HttpMethod("POST"), data);
var response = await _request<CreateIndexResponse>(url, new HttpMethod("PUT"), data);
return InvokeResult.Succeeding<CreateIndexResponse>(response);
}
catch (Exception ex)
@ -164,7 +161,25 @@ namespace ZeroLevel.Qdrant
/// <summary>
/// There is a method for retrieving points by their ids.
/// </summary>
public async Task<InvokeResult<PointResponse>> Points(string collection_name, long[] ids)
public async Task<InvokeResult<PointResponse>> GetPoint(string collection_name, long id)
{
try
{
string url = $"/collections/{collection_name}/points/{id}";
var response = await _request<PointResponse>(url, new HttpMethod("GET"), null);
return InvokeResult.Succeeding<PointResponse>(response);
}
catch (Exception ex)
{
Log.Error(ex, $"[QdrantClient.Points] Collection name: {collection_name}.");
return InvokeResult.Fault<PointResponse>($"[QdrantClient.GetPoint] Collection name: {collection_name}. Point ID: {id}\r\n{ex.ToString()}");
}
}
/// <summary>
/// There is a method for retrieving points by their ids.
/// </summary>
public async Task<InvokeResult<PointsResponse>> GetPoints(string collection_name, long[] ids)
{
try
{
@ -172,13 +187,13 @@ namespace ZeroLevel.Qdrant
var json = JsonConvert.SerializeObject(points);
var data = new StringContent(json, Encoding.UTF8, "application/json");
string url = $"/collections/{collection_name}/points";
var response = await _request<PointResponse>(url, new HttpMethod("POST"), data);
return InvokeResult.Succeeding<PointResponse>(response);
var response = await _request<PointsResponse>(url, new HttpMethod("POST"), data);
return InvokeResult.Succeeding<PointsResponse>(response);
}
catch (Exception ex)
{
Log.Error(ex, $"[QdrantClient.Points] Collection name: {collection_name}.");
return InvokeResult.Fault<PointResponse>($"[QdrantClient.Points] Collection name: {collection_name}.\r\n{ex.ToString()}");
return InvokeResult.Fault<PointsResponse>($"[QdrantClient.GetPoints] Collection name: {collection_name}.\r\n{ex.ToString()}");
}
}
@ -208,43 +223,20 @@ namespace ZeroLevel.Qdrant
/// <summary>
/// Record-oriented of creating batches
/// </summary>
public async Task<InvokeResult<PointsOperationResponse>> PointsUpload<T>(string collection_name, UpsertPoint<T>[] points)
public async Task<InvokeResult<PointsOperationResponse>> UpsertPoints<T>(string collection_name, PointsUpsertRequest<T> points)
{
try
{
var points_request = new PointsUploadRequest<T> { upsert_points = new UpsertPoints<T> { points = points } };
var json = points_request.ToJSON();
var json = points.ToJSON();
var data = new StringContent(json, Encoding.UTF8, "application/json");
var url = $"/collections/{collection_name}";
var response = await _request<PointsOperationResponse>(url, new HttpMethod("POST"), data);
var url = $"/collections/{collection_name}/points";
var response = await _request<PointsOperationResponse>(url, new HttpMethod("PUT"), data);
return InvokeResult.Succeeding<PointsOperationResponse>(response);
}
catch (Exception ex)
{
Log.Error(ex, $"[QdrantClient.Points] Collection name: {collection_name}.");
return InvokeResult.Fault<PointsOperationResponse>($"[QdrantClient.Points] Collection name: {collection_name}.\r\n{ex.ToString()}");
}
}
/// <summary>
/// Column-oriented of creating batches
/// </summary>
public async Task<InvokeResult<PointsOperationResponse>> PointsColumnUpload<T>(string collection_name, ColumnPoints<T> points)
{
try
{
var points_request = new PointsColumnUploadRequest<T> { upsert_points = new UpsertColumnPoints<T> { batch = points } };
var json = points_request.ToJSON();
var data = new StringContent(json, Encoding.UTF8, "application/json");
var url = $"/collections/{collection_name}";
var response = await _request<PointsOperationResponse>(url, new HttpMethod("POST"), data);
return InvokeResult.Succeeding<PointsOperationResponse>(response);
}
catch (Exception ex)
{
Log.Error(ex, $"[QdrantClient.Points] Collection name: {collection_name}.");
return InvokeResult.Fault<PointsOperationResponse>($"[QdrantClient.Points] Collection name: {collection_name}.\r\n{ex.ToString()}");
Log.Error(ex, $"[QdrantClient.UpsertPoints] Collection name: {collection_name}.");
return InvokeResult.Fault<PointsOperationResponse>($"[QdrantClient.UpsertPoints] Collection name: {collection_name}.\r\n{ex.ToString()}");
}
}
/// <summary>

11
ZeroLevel.Qdrant/Services/QdrantJsonConverter.cs
Unescape View File

@ -11,7 +11,7 @@ using ZeroLevel.Services.Serialization;
namespace ZeroLevel.Qdrant.Services
{
public class QdrantJsonConverter<T>
public static class QdrantJsonConverter<T>
{
private static string KeywordToString(IMemberInfo member, object v)
{
@ -36,7 +36,12 @@ namespace ZeroLevel.Qdrant.Services
private const string GEO_TYPE = "geo";
private const string FLOAT_TYPE = "float";
private const string INTEGER_TYPE = "integer";
public string ToJson(T value)
public static string ConvertToJson(float[] vector)
{
return "[" + string.Join(", ", vector.Select(f => f.ConvertToString())) + "]";
}
public static string ConvertToJson(T value)
{
var json = new StringBuilder();
@ -154,7 +159,7 @@ namespace ZeroLevel.Qdrant.Services
return json.ToString();
}
public IEnumerable<object> E(IEnumerable e)
private static IEnumerable<object> E(IEnumerable e)
{
if (e != null)
{

4
ZeroLevel/Services/Reflection/TypeHelpers.cs
Unescape View File

@ -47,6 +47,10 @@ namespace ZeroLevel.Services.Reflection
return false;
}
}
<<<<<<< Updated upstream
=======
>>>>>>> Stashed changes
public static bool IsNumericTypeWithFloating(Type type)
{
switch (Type.GetTypeCode(type))

Write Preview
Loading…
Cancel
Save

Powered by TurnKey Linux.