|
|
@ -1,4 +1,5 @@
|
|
|
|
using System.Collections.Generic;
|
|
|
|
using System.Collections.Concurrent;
|
|
|
|
|
|
|
|
using System.Collections.Generic;
|
|
|
|
using System.Threading;
|
|
|
|
using System.Threading;
|
|
|
|
using ZeroLevel.Services.Serialization;
|
|
|
|
using ZeroLevel.Services.Serialization;
|
|
|
|
|
|
|
|
|
|
|
@ -10,16 +11,14 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
internal class TrieNode
|
|
|
|
internal class TrieNode
|
|
|
|
: IBinarySerializable
|
|
|
|
: IBinarySerializable
|
|
|
|
{
|
|
|
|
{
|
|
|
|
public char Key;
|
|
|
|
|
|
|
|
public uint? Value;
|
|
|
|
public uint? Value;
|
|
|
|
public TrieNode Parent;
|
|
|
|
public TrieNode Parent;
|
|
|
|
public List<TrieNode> Children;
|
|
|
|
public ConcurrentDictionary<char, TrieNode> Children;
|
|
|
|
|
|
|
|
|
|
|
|
public TrieNode() { }
|
|
|
|
public TrieNode() { }
|
|
|
|
public TrieNode(TrieNode parent) { Parent = parent; }
|
|
|
|
public TrieNode(TrieNode parent) { Parent = parent; }
|
|
|
|
public void Deserialize(IBinaryReader reader)
|
|
|
|
public void Deserialize(IBinaryReader reader)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
this.Key = reader.ReadChar();
|
|
|
|
|
|
|
|
if (reader.ReadBoolean())
|
|
|
|
if (reader.ReadBoolean())
|
|
|
|
{
|
|
|
|
{
|
|
|
|
this.Value = reader.ReadUInt32();
|
|
|
|
this.Value = reader.ReadUInt32();
|
|
|
@ -28,12 +27,11 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
{
|
|
|
|
{
|
|
|
|
this.Value = null;
|
|
|
|
this.Value = null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
this.Children = reader.ReadCollection<TrieNode>();
|
|
|
|
this.Children = reader.ReadDictionaryAsConcurrent<char, TrieNode>();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public void Serialize(IBinaryWriter writer)
|
|
|
|
public void Serialize(IBinaryWriter writer)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
writer.WriteChar(this.Key);
|
|
|
|
|
|
|
|
if (this.Value.HasValue)
|
|
|
|
if (this.Value.HasValue)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
writer.WriteBoolean(true);
|
|
|
|
writer.WriteBoolean(true);
|
|
|
@ -49,7 +47,7 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
else
|
|
|
|
{
|
|
|
|
{
|
|
|
|
writer.WriteCollection<TrieNode>(this.Children);
|
|
|
|
writer.WriteDictionary<char, TrieNode>(this.Children);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
internal TrieNode Append(string word, int index, bool reverse)
|
|
|
|
internal TrieNode Append(string word, int index, bool reverse)
|
|
|
@ -65,40 +63,22 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if (this.Children == null)
|
|
|
|
if (this.Children == null)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
this.Children = new List<TrieNode>();
|
|
|
|
this.Children = new ConcurrentDictionary<char, TrieNode>();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for (int i = 0; i < Children.Count; i++)
|
|
|
|
if (!this.Children.ContainsKey(word[index]))
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if (Children[i].Key == word[index])
|
|
|
|
this.Children.TryAdd(word[index], new TrieNode(this));
|
|
|
|
{
|
|
|
|
|
|
|
|
return Children[i].Append(word, index + 1, reverse);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
var tn = new TrieNode(reverse ? this : null) { Key = word[index] };
|
|
|
|
return Children[word[index]].Append(word, index + 1, reverse);
|
|
|
|
Children.Add(tn);
|
|
|
|
|
|
|
|
return tn.Append(word, index + 1, reverse);
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return null;
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
internal uint? GetKey(string word, int index)
|
|
|
|
internal uint? GetKey(string word, int index)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if (word.Length == index + 1)
|
|
|
|
if (this.Children?.ContainsKey(word[index]) ?? false)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
return this.Value;
|
|
|
|
if (word.Length == index + 1) return this.Children[word[index]].Value;
|
|
|
|
}
|
|
|
|
return this.Children[word[index]].GetKey(word, index + 1);
|
|
|
|
else
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
if (this.Children == null)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
this.Children = new List<TrieNode>();
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
for (int i = 0; i < Children.Count; i++)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
if (Children[i].Key == word[index])
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
return Children[i].GetKey(word, index + 1);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return null;
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -114,7 +94,7 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
{
|
|
|
|
{
|
|
|
|
foreach (var child in this.Children)
|
|
|
|
foreach (var child in this.Children)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
child.RebuildReverseIndex(this, index);
|
|
|
|
child.Value.RebuildReverseIndex(this, index);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -126,17 +106,22 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
{
|
|
|
|
{
|
|
|
|
foreach (var child in this.Children)
|
|
|
|
foreach (var child in this.Children)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
child.DestroyReverseIndex();
|
|
|
|
child.Value.DestroyReverseIndex();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
internal List<TrieNode> _roots;
|
|
|
|
internal TrieNode _root;
|
|
|
|
private int _word_index = 0;
|
|
|
|
private int _word_index = 0;
|
|
|
|
private bool _use_reverse_index;
|
|
|
|
private bool _use_reverse_index;
|
|
|
|
|
|
|
|
|
|
|
|
private Dictionary<uint, TrieNode> _reverse_index;
|
|
|
|
private Dictionary<uint, TrieNode> _reverse_index;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public Trie() : this(false)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public Trie(bool reverse_index = false)
|
|
|
|
public Trie(bool reverse_index = false)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
_use_reverse_index = reverse_index;
|
|
|
|
_use_reverse_index = reverse_index;
|
|
|
@ -144,7 +129,7 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
{
|
|
|
|
{
|
|
|
|
_reverse_index = new Dictionary<uint, TrieNode>();
|
|
|
|
_reverse_index = new Dictionary<uint, TrieNode>();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
_roots = new List<TrieNode>();
|
|
|
|
_root = new TrieNode();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public void ToggleReverseIndex(bool enabled)
|
|
|
|
public void ToggleReverseIndex(bool enabled)
|
|
|
@ -163,37 +148,13 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
public void Append(string word)
|
|
|
|
public void Append(string word)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if (word.Length == 0) return;
|
|
|
|
if (word.Length == 0) return;
|
|
|
|
bool found = false;
|
|
|
|
var node = _root.Append(word, 0, _use_reverse_index);
|
|
|
|
for (int i = 0; i < _roots.Count; i++)
|
|
|
|
if (node != null)
|
|
|
|
{
|
|
|
|
|
|
|
|
if (_roots[i].Key == word[0])
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
var node = _roots[i].Append(word, 1, _use_reverse_index);
|
|
|
|
|
|
|
|
Thread.MemoryBarrier();
|
|
|
|
|
|
|
|
if (node != null)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
node.Value = (uint)Interlocked.Increment(ref _word_index);
|
|
|
|
|
|
|
|
if (_use_reverse_index)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
_reverse_index.Add(node.Value.Value, node);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
found = true;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!found)
|
|
|
|
|
|
|
|
{
|
|
|
|
{
|
|
|
|
var tn = new TrieNode { Key = word[0] };
|
|
|
|
|
|
|
|
_roots.Add(tn);
|
|
|
|
|
|
|
|
var node = tn.Append(word, 1, _use_reverse_index);
|
|
|
|
|
|
|
|
Thread.MemoryBarrier();
|
|
|
|
|
|
|
|
node.Value = (uint)Interlocked.Increment(ref _word_index);
|
|
|
|
node.Value = (uint)Interlocked.Increment(ref _word_index);
|
|
|
|
if (node != null)
|
|
|
|
if (_use_reverse_index)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if (_use_reverse_index)
|
|
|
|
_reverse_index.Add(node.Value.Value, node);
|
|
|
|
{
|
|
|
|
|
|
|
|
_reverse_index.Add(node.Value.Value, node);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -201,55 +162,27 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
public uint? Key(string word)
|
|
|
|
public uint? Key(string word)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if (word?.Length == 0) return null;
|
|
|
|
if (word?.Length == 0) return null;
|
|
|
|
for (int i = 0; i < _roots.Count; i++)
|
|
|
|
return _root.GetKey(word, 0);
|
|
|
|
{
|
|
|
|
|
|
|
|
if (_roots[i].Key == word[0])
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
if (word.Length == 1)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
return _roots[i].Value;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
else
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
return _roots[i].GetKey(word, 1);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return null;
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public bool Contains(string word)
|
|
|
|
public bool Contains(string word)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if (word?.Length == 0) return false;
|
|
|
|
if (word?.Length == 0) return false;
|
|
|
|
for (int i = 0; i < _roots.Count; i++)
|
|
|
|
return _root.GetKey(word, 0).HasValue;
|
|
|
|
{
|
|
|
|
|
|
|
|
if (_roots[i].Key == word[0])
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
if (word.Length == 1)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
return _roots[i].Value.HasValue;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
else
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
return _roots[i].GetKey(word, 1).HasValue;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public void Serialize(IBinaryWriter writer)
|
|
|
|
public void Serialize(IBinaryWriter writer)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
writer.WriteInt32(this._word_index);
|
|
|
|
writer.WriteInt32(this._word_index);
|
|
|
|
writer.WriteBoolean(this._use_reverse_index);
|
|
|
|
writer.WriteBoolean(this._use_reverse_index);
|
|
|
|
writer.WriteCollection<TrieNode>(this._roots);
|
|
|
|
writer.Write<TrieNode>(this._root);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public void Deserialize(IBinaryReader reader)
|
|
|
|
public void Deserialize(IBinaryReader reader)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
this._word_index = reader.ReadInt32();
|
|
|
|
this._word_index = reader.ReadInt32();
|
|
|
|
this._use_reverse_index = reader.ReadBoolean();
|
|
|
|
this._use_reverse_index = reader.ReadBoolean();
|
|
|
|
this._roots = reader.ReadCollection<TrieNode>();
|
|
|
|
this._root = reader.Read<TrieNode>();
|
|
|
|
RebuildReverseIndex();
|
|
|
|
RebuildReverseIndex();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -261,10 +194,7 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
{
|
|
|
|
{
|
|
|
|
_reverse_index = new Dictionary<uint, TrieNode>();
|
|
|
|
_reverse_index = new Dictionary<uint, TrieNode>();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
foreach (var node in _roots)
|
|
|
|
_root.RebuildReverseIndex(null, _reverse_index);
|
|
|
|
{
|
|
|
|
|
|
|
|
node.RebuildReverseIndex(null, _reverse_index);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -275,10 +205,7 @@ namespace ZeroLevel.Services.Semantic
|
|
|
|
_reverse_index.Clear();
|
|
|
|
_reverse_index.Clear();
|
|
|
|
_reverse_index = null;
|
|
|
|
_reverse_index = null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
foreach (var node in _roots)
|
|
|
|
_root.DestroyReverseIndex();
|
|
|
|
{
|
|
|
|
|
|
|
|
node.DestroyReverseIndex();
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|