diff --git a/ZeroLevel.UnitTests/BloomFilterTest.cs b/ZeroLevel.UnitTests/BloomFilterTest.cs index 4495a0c..f7894b0 100644 --- a/ZeroLevel.UnitTests/BloomFilterTest.cs +++ b/ZeroLevel.UnitTests/BloomFilterTest.cs @@ -55,7 +55,7 @@ namespace ZeroLevel.UnitTests { if (false == lines.Contains(line)) { - collision_count++; + collision_count++; } } } @@ -67,7 +67,7 @@ namespace ZeroLevel.UnitTests public void HyperBloomBloomFilterTest() { // Arrange - var size = 100000; + var size = 1000000; var lines = new HashSet(size); var lines_another = new HashSet(size); for (int i = 0; i < size; i++) @@ -75,7 +75,7 @@ namespace ZeroLevel.UnitTests lines.Add(RandomString(i % 9 + 5)); lines_another.Add(RandomString(i % 9 + 5)); } - var bloom = new HyperBloomBloom(16536 * 1024, true); + var bloom = new HyperBloomBloom(64, 16536 * 1024, true); // Act var sw = new Stopwatch(); sw.Start(); diff --git a/ZeroLevel/DataStructures/HyperBloomBloom.cs b/ZeroLevel/DataStructures/HyperBloomBloom.cs index 545e45e..2e48a31 100644 --- a/ZeroLevel/DataStructures/HyperBloomBloom.cs +++ b/ZeroLevel/DataStructures/HyperBloomBloom.cs @@ -1,38 +1,33 @@ -using System; -using System.Collections.Generic; +using ZeroLevel.Services.HashFunctions; namespace ZeroLevel.DataStructures { public class HyperBloomBloom { - private BloomFilter _trash; - private Dictionary _shardes = new Dictionary(); + private IHash _shardHash = new XXHashUnsafe(); + private BloomFilter[] _shardes; - public HyperBloomBloom(int bit_size, bool use_reverse) + public HyperBloomBloom(int shardes_size, int bit_size, bool use_reverse) { - _trash = new BloomFilter(bit_size, use_reverse); - foreach (var ch in "abcdefghijklmnopqrstuvwxyz0123456789-") + _shardes = new BloomFilter[shardes_size]; + for (int i = 0; i < shardes_size; i++) { - _shardes.Add(ch, new BloomFilter(bit_size, use_reverse)); + _shardes[i] = new BloomFilter(bit_size, use_reverse); } } public void Add(string item) { if (item == null || item.Length == 0) return; - var k = Char.ToLowerInvariant(item[0]); - BloomFilter filter; - if (_shardes.TryGetValue(k, out filter) == false) filter = _trash; - filter.Add(item); + int index = (int)(_shardHash.Hash(item) % _shardes.Length); + _shardes[index].Add(item); } public bool Contains(string item) { if (item == null || item.Length == 0) return true; - var k = Char.ToLowerInvariant(item[0]); - BloomFilter filter; - if (_shardes.TryGetValue(k, out filter) == false) filter = _trash; - return filter.Contains(item); + int index = (int)(_shardHash.Hash(item) % _shardes.Length); + return _shardes[index].Contains(item); } /// /// true if added, false if already exists @@ -40,10 +35,8 @@ namespace ZeroLevel.DataStructures public bool TryAdd(string item) { if (item == null || item.Length == 0) return false; - var k = Char.ToLowerInvariant(item[0]); - BloomFilter filter; - if (_shardes.TryGetValue(k, out filter) == false) filter = _trash; - return filter.TryAdd(item); + int index = (int)(_shardHash.Hash(item) % _shardes.Length); + return _shardes[index].TryAdd(item); } } }