Add Ngramm builder

pull/1/head
Ogoun 6 years ago
parent 1a086cec04
commit 0b4de16640

@ -0,0 +1,81 @@
using System;
using System.Collections.Generic;
using ZeroLevel.Services.Collections;
/*
Example with text lines
var freg_dict = BuildNGramm(File.ReadAllLines("samples.txt"), line => _provider.ExtractLexTokens(line).Where(w => StopWords.IsStopWord(w.Token) == false).Select(t => t.Token.ToLowerInvariant()), 2);
Example with sentences
var freg_dict = BuildNGramm<Sentence>(TAStringReader.ReadSentenses(File.ReadAllText("samples.txt")), sent => sent.Tokens.Select(t => t.Value).Where(w => StopWords.IsStopWord(w) == false), 3);
*/
namespace ZeroLevel.Services.Semantic.Helpers
{
public static class NGramms
{
public static Dictionary<string, int> BuildNGramm(IEnumerable<string> input, Func<string, IEnumerable<string>> tokenizer, int N)
{
var ngramms = new Dictionary<string, int>();
var arr = new FixSizeQueue<string>(N);
foreach (var line in input)
{
foreach (var token in tokenizer(line))
{
arr.Push(token);
if (arr.Count == N)
{
var currentPrase = string.Join(" ", arr.Dump());
if (ngramms.ContainsKey(currentPrase)) ngramms[currentPrase]++;
else ngramms.Add(currentPrase, 1);
}
}
while (arr.Count > 0)
arr.Take();
}
return ngramms;
}
public static Dictionary<string, int> GetUnigramms(IEnumerable<string> input, Func<string, IEnumerable<string>> tokenizer)
=> BuildNGramm(input, tokenizer, 1);
public static Dictionary<string, int> GetBigramms(IEnumerable<string> input, Func<string, IEnumerable<string>> tokenizer)
=> BuildNGramm(input, tokenizer, 2);
public static Dictionary<string, int> GetTrigramms(IEnumerable<string> input, Func<string, IEnumerable<string>> tokenizer)
=> BuildNGramm(input, tokenizer, 3);
public static Dictionary<string, int> BuildNGramm<T>(IEnumerable<T> input, Func<T, IEnumerable<string>> tokenizer, int N)
{
var ngramms = new Dictionary<string, int>();
var arr = new FixSizeQueue<string>(N);
foreach (var item in input)
{
foreach (var token in tokenizer(item))
{
arr.Push(token);
if (arr.Count == N)
{
var currentPrase = string.Join(" ", arr.Dump());
if (ngramms.ContainsKey(currentPrase)) ngramms[currentPrase]++;
else ngramms.Add(currentPrase, 1);
}
}
while (arr.Count > 0)
arr.Take();
}
return ngramms;
}
public static Dictionary<string, int> GetUnigramms<T>(IEnumerable<T> input, Func<T, IEnumerable<string>> tokenizer)
=> BuildNGramm(input, tokenizer, 1);
public static Dictionary<string, int> GetBigramms<T>(IEnumerable<T> input, Func<T, IEnumerable<string>> tokenizer)
=> BuildNGramm(input, tokenizer, 2);
public static Dictionary<string, int> GetTrigramms<T>(IEnumerable<T> input, Func<T, IEnumerable<string>> tokenizer)
=> BuildNGramm(input, tokenizer, 3);
}
}

@ -294,6 +294,7 @@
<Compile Include="Services\Semantic\Contracts\Languages.cs" /> <Compile Include="Services\Semantic\Contracts\Languages.cs" />
<Compile Include="Services\Semantic\Contracts\LexToken.cs" /> <Compile Include="Services\Semantic\Contracts\LexToken.cs" />
<Compile Include="Services\Semantic\Contracts\WordToken.cs" /> <Compile Include="Services\Semantic\Contracts\WordToken.cs" />
<Compile Include="Services\Semantic\Helpers\NGramms.cs" />
<Compile Include="Services\Semantic\Helpers\StopWords.cs" /> <Compile Include="Services\Semantic\Helpers\StopWords.cs" />
<Compile Include="Services\Semantic\Helpers\TextAnalizer.cs" /> <Compile Include="Services\Semantic\Helpers\TextAnalizer.cs" />
<Compile Include="Services\Semantic\LexProvider.cs" /> <Compile Include="Services\Semantic\LexProvider.cs" />

Loading…
Cancel
Save

Powered by TurnKey Linux.