HyperBloomBloom fix

pull/1/head
Ogoun 5 years ago
parent c3619be068
commit 0817ed767c

@ -67,7 +67,7 @@ namespace ZeroLevel.UnitTests
public void HyperBloomBloomFilterTest() public void HyperBloomBloomFilterTest()
{ {
// Arrange // Arrange
var size = 100000; var size = 1000000;
var lines = new HashSet<string>(size); var lines = new HashSet<string>(size);
var lines_another = new HashSet<string>(size); var lines_another = new HashSet<string>(size);
for (int i = 0; i < size; i++) for (int i = 0; i < size; i++)
@ -75,7 +75,7 @@ namespace ZeroLevel.UnitTests
lines.Add(RandomString(i % 9 + 5)); lines.Add(RandomString(i % 9 + 5));
lines_another.Add(RandomString(i % 9 + 5)); lines_another.Add(RandomString(i % 9 + 5));
} }
var bloom = new HyperBloomBloom(16536 * 1024, true); var bloom = new HyperBloomBloom(64, 16536 * 1024, true);
// Act // Act
var sw = new Stopwatch(); var sw = new Stopwatch();
sw.Start(); sw.Start();

@ -1,38 +1,33 @@
using System; using ZeroLevel.Services.HashFunctions;
using System.Collections.Generic;
namespace ZeroLevel.DataStructures namespace ZeroLevel.DataStructures
{ {
public class HyperBloomBloom public class HyperBloomBloom
{ {
private BloomFilter _trash; private IHash _shardHash = new XXHashUnsafe();
private Dictionary<char, BloomFilter> _shardes = new Dictionary<char, BloomFilter>(); private BloomFilter[] _shardes;
public HyperBloomBloom(int bit_size, bool use_reverse) public HyperBloomBloom(int shardes_size, int bit_size, bool use_reverse)
{ {
_trash = new BloomFilter(bit_size, use_reverse); _shardes = new BloomFilter[shardes_size];
foreach (var ch in "abcdefghijklmnopqrstuvwxyz0123456789-") for (int i = 0; i < shardes_size; i++)
{ {
_shardes.Add(ch, new BloomFilter(bit_size, use_reverse)); _shardes[i] = new BloomFilter(bit_size, use_reverse);
} }
} }
public void Add(string item) public void Add(string item)
{ {
if (item == null || item.Length == 0) return; if (item == null || item.Length == 0) return;
var k = Char.ToLowerInvariant(item[0]); int index = (int)(_shardHash.Hash(item) % _shardes.Length);
BloomFilter filter; _shardes[index].Add(item);
if (_shardes.TryGetValue(k, out filter) == false) filter = _trash;
filter.Add(item);
} }
public bool Contains(string item) public bool Contains(string item)
{ {
if (item == null || item.Length == 0) return true; if (item == null || item.Length == 0) return true;
var k = Char.ToLowerInvariant(item[0]); int index = (int)(_shardHash.Hash(item) % _shardes.Length);
BloomFilter filter; return _shardes[index].Contains(item);
if (_shardes.TryGetValue(k, out filter) == false) filter = _trash;
return filter.Contains(item);
} }
/// <summary> /// <summary>
/// true if added, false if already exists /// true if added, false if already exists
@ -40,10 +35,8 @@ namespace ZeroLevel.DataStructures
public bool TryAdd(string item) public bool TryAdd(string item)
{ {
if (item == null || item.Length == 0) return false; if (item == null || item.Length == 0) return false;
var k = Char.ToLowerInvariant(item[0]); int index = (int)(_shardHash.Hash(item) % _shardes.Length);
BloomFilter filter; return _shardes[index].TryAdd(item);
if (_shardes.TryGetValue(k, out filter) == false) filter = _trash;
return filter.TryAdd(item);
} }
} }
} }

Loading…
Cancel
Save

Powered by TurnKey Linux.