From 03fa1f503b5a65054f41534fd472f0c59f4495d6 Mon Sep 17 00:00:00 2001 From: Ogoun Date: Wed, 6 Oct 2021 02:54:23 +0300 Subject: [PATCH] Add the Scroll method --- ZeroLevel.Qdrant/Models/Point.cs | 9 ++++ .../Models/Requests/PointsRequest.cs | 2 + .../Models/Requests/ScrollRequest.cs | 51 +++++++++++++++++++ .../Models/Responces/PointResponse.cs | 7 --- .../Models/Responces/ScrollResponse.cs | 15 ++++++ ZeroLevel.Qdrant/QdrantClient.cs | 27 +++++++++- 6 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 ZeroLevel.Qdrant/Models/Point.cs create mode 100644 ZeroLevel.Qdrant/Models/Requests/ScrollRequest.cs create mode 100644 ZeroLevel.Qdrant/Models/Responces/ScrollResponse.cs diff --git a/ZeroLevel.Qdrant/Models/Point.cs b/ZeroLevel.Qdrant/Models/Point.cs new file mode 100644 index 0000000..64f0121 --- /dev/null +++ b/ZeroLevel.Qdrant/Models/Point.cs @@ -0,0 +1,9 @@ +namespace ZeroLevel.Qdrant.Models +{ + public sealed class Point + { + public long id { get; set; } + public dynamic payload; + public double[] vector; + } +} diff --git a/ZeroLevel.Qdrant/Models/Requests/PointsRequest.cs b/ZeroLevel.Qdrant/Models/Requests/PointsRequest.cs index 9e5cd00..9fe7799 100644 --- a/ZeroLevel.Qdrant/Models/Requests/PointsRequest.cs +++ b/ZeroLevel.Qdrant/Models/Requests/PointsRequest.cs @@ -3,5 +3,7 @@ internal sealed class PointsRequest { public long[] ids { get; set; } + /*public bool with_payload { get; set; } = true; + public bool with_vector { get; set; } = true;*/ } } diff --git a/ZeroLevel.Qdrant/Models/Requests/ScrollRequest.cs b/ZeroLevel.Qdrant/Models/Requests/ScrollRequest.cs new file mode 100644 index 0000000..69a0162 --- /dev/null +++ b/ZeroLevel.Qdrant/Models/Requests/ScrollRequest.cs @@ -0,0 +1,51 @@ +using System; +using System.Text; +using ZeroLevel.Qdrant.Models.Filters; + +namespace ZeroLevel.Qdrant.Models.Requests +{ + internal sealed class ScrollRequest + { + public Filter Filter { get; set; } + public bool WithPayload { get; set; } = true; + public bool WithVector { get; set; } = true; + public long Limit { get; set; } + public long Offset { get; set; } + + /* + { + "filter": { + "must": [ + { "has_id": [0, 3, 100] } + ] + }, + "limit": 10000, + "offset": 0, + "with_payload": false, + "with_vector": true +} + */ + + public string ToJson() + { + var json = new StringBuilder(); + json.Append("{"); + if (Filter == null || Filter.IsEmpty) + { + throw new ArgumentException("Filter must not by null or empty"); + } + else + { + json.Append(Filter.ToJSON()); + json.Append(','); + } + json.Append($"\"limit\": {Limit},"); + json.Append($"\"offset\": {Offset},"); + json.Append($"\"with_payload\": {WithPayload.ToString().ToLowerInvariant()},"); + json.Append($"\"with_vector\": {WithVector.ToString().ToLowerInvariant()}"); + json.Append("}"); + return json.ToString(); + + } + } +} diff --git a/ZeroLevel.Qdrant/Models/Responces/PointResponse.cs b/ZeroLevel.Qdrant/Models/Responces/PointResponse.cs index b932fb1..54c92b1 100644 --- a/ZeroLevel.Qdrant/Models/Responces/PointResponse.cs +++ b/ZeroLevel.Qdrant/Models/Responces/PointResponse.cs @@ -1,12 +1,5 @@ namespace ZeroLevel.Qdrant.Models.Responces { - public sealed class Point - { - public long id { get; set; } - public dynamic payload; - public double[] vector; - } - public sealed class PointResponse { public Point[] result { get; set; } diff --git a/ZeroLevel.Qdrant/Models/Responces/ScrollResponse.cs b/ZeroLevel.Qdrant/Models/Responces/ScrollResponse.cs new file mode 100644 index 0000000..fb5b00e --- /dev/null +++ b/ZeroLevel.Qdrant/Models/Responces/ScrollResponse.cs @@ -0,0 +1,15 @@ +namespace ZeroLevel.Qdrant.Models.Responces +{ + public sealed class ScrollResult + { + public Point[] points { get; set; } + public long? next_page_offset { get; set; } + } + + public sealed class ScrollResponse + { + public ScrollResult result { get; set; } + public string status { get; set; } + public float time { get; set; } + } +} diff --git a/ZeroLevel.Qdrant/QdrantClient.cs b/ZeroLevel.Qdrant/QdrantClient.cs index 0e69c40..11198b5 100644 --- a/ZeroLevel.Qdrant/QdrantClient.cs +++ b/ZeroLevel.Qdrant/QdrantClient.cs @@ -171,8 +171,7 @@ namespace ZeroLevel.Qdrant var points = new PointsRequest { ids = ids }; var json = JsonConvert.SerializeObject(points); var data = new StringContent(json, Encoding.UTF8, "application/json"); - var url = $"/collections/{collection_name}/points"; - + string url = $"/collections/{collection_name}/points"; var response = await _request(url, new HttpMethod("POST"), data); return InvokeResult.Succeeding(response); } @@ -182,6 +181,30 @@ namespace ZeroLevel.Qdrant return InvokeResult.Fault($"[QdrantClient.Points] Collection name: {collection_name}.\r\n{ex.ToString()}"); } } + + /// + /// There is a method for retrieving points by their ids. + /// + public async Task> Scroll(string collection_name, Filter filter, long limit, long offset = 0, bool with_vector = true, bool with_payload = true) + { + try + { + var scroll = new ScrollRequest { Filter = filter, Limit = limit, Offset = offset, WithPayload = with_payload, WithVector = with_vector }; + var json = scroll.ToJson(); + var data = new StringContent(json, Encoding.UTF8, "application/json"); + string url = url = $"/collections/{collection_name}/points/scroll"; + + var response = await _request(url, new HttpMethod("POST"), data); + return InvokeResult.Succeeding(response); + } + catch (Exception ex) + { + Log.Error(ex, $"[QdrantClient.Scroll] Collection name: {collection_name}."); + return InvokeResult.Fault($"[QdrantClient.Scroll] Collection name: {collection_name}.\r\n{ex.ToString()}"); + } + } + + /// /// Record-oriented of creating batches ///