diff --git a/ZeroLevel.UnitTests/SemanticTests.cs b/ZeroLevel.UnitTests/SemanticTests.cs index ba16b1d..b5d75da 100644 --- a/ZeroLevel.UnitTests/SemanticTests.cs +++ b/ZeroLevel.UnitTests/SemanticTests.cs @@ -29,6 +29,16 @@ namespace ZeroLevel.UnitTests { Assert.True(string.CompareOrdinal(test[i], terms[i]) == 0); } + + Assert.False(WordTokenizer.Tokenize(string.Empty).Any()); + Assert.False(WordTokenizer.Tokenize(null).Any()); + Assert.False(WordTokenizer.Tokenize(" ").Any()); + Assert.False(WordTokenizer.Tokenize(" ").Any()); + Assert.False(WordTokenizer.Tokenize(" 1 ").Any()); + Assert.False(WordTokenizer.Tokenize("1 1").Any()); + Assert.False(WordTokenizer.Tokenize(" 1 1 ").Any()); + Assert.False(WordTokenizer.Tokenize(" 12aa 1a3 ").Any()); + Assert.False(WordTokenizer.Tokenize(" __a1 _a1 ").Any()); } } } diff --git a/ZeroLevel/Services/Semantic/Trie.cs b/ZeroLevel/Services/Semantic/Trie.cs index 31effa7..d13367a 100644 --- a/ZeroLevel/Services/Semantic/Trie.cs +++ b/ZeroLevel/Services/Semantic/Trie.cs @@ -15,7 +15,7 @@ namespace ZeroLevel.Services.Semantic public char? Key; // setted only with rebuild index public uint? Value; public TrieNode Parent; - public ConcurrentDictionary Children; + public Dictionary Children; public TrieNode() { } public TrieNode(TrieNode parent) { Parent = parent; } @@ -29,7 +29,7 @@ namespace ZeroLevel.Services.Semantic { this.Value = null; } - this.Children = reader.ReadDictionaryAsConcurrent(); + this.Children = reader.ReadDictionary(); } public void Serialize(IBinaryWriter writer) @@ -52,7 +52,7 @@ namespace ZeroLevel.Services.Semantic writer.WriteDictionary(this.Children); } } - internal TrieNode Append(string word, int index, bool reverse) + internal TrieNode Append(string word, int index) { if (word.Length == index) { @@ -65,13 +65,13 @@ namespace ZeroLevel.Services.Semantic { if (this.Children == null) { - this.Children = new ConcurrentDictionary(); + this.Children = new Dictionary(); } if (!this.Children.ContainsKey(word[index])) { - this.Children.TryAdd(word[index], new TrieNode(this)); + this.Children.Add(word[index], new TrieNode(this)); } - return Children[word[index]].Append(word, index + 1, reverse); + return Children[word[index]].Append(word, index + 1); } return null; } @@ -154,7 +154,7 @@ namespace ZeroLevel.Services.Semantic public void Append(string word) { if (word.Length == 0) return; - var node = _root.Append(word, 0, _use_reverse_index); + var node = _root.Append(word, 0); if (node != null) { node.Value = (uint)Interlocked.Increment(ref _word_index); diff --git a/ZeroLevel/Services/Semantic/WordTokenizer.cs b/ZeroLevel/Services/Semantic/WordTokenizer.cs index 17490d2..b5fa5b0 100644 --- a/ZeroLevel/Services/Semantic/WordTokenizer.cs +++ b/ZeroLevel/Services/Semantic/WordTokenizer.cs @@ -15,7 +15,7 @@ namespace ZeroLevel.Services.Semantic var buffer = _pool.Allocate(); try { - for (int i = 0; i < text.Length; i++) + for (int i = 0; i < text?.Length; i++) { if (first && Char.IsLetter(text[i])) {