/*
* Port of Snowball stemmers on C#
* Original stemmers can be found on http://snowball.tartarus.org
* Licence still BSD: http://snowball.tartarus.org/license.php
*
* Most of stemmers are ported from Java by Iveonik Systems ltd. (www.iveonik.com)
*
* German stemmer's port found on SourceForge site
*/
using System;
using System.Threading;
using ZeroLevel.Services.Semantic;
namespace Iveonik.Stemmers
{
public class GermanStemmer : ILexer
{
private char[] word_buffer;
private int STR_SIZE, R1, R2;
private int BUFFER_SIZE;
private const int INC = 20; // I found out this is optimal word string size
public GermanStemmer()
{
SetInitState();
}
private void SetInitState()
{
R1 = R2 = -1; STR_SIZE = 0;
BUFFER_SIZE = INC;
word_buffer = null;
word_buffer = new char[BUFFER_SIZE];
}
private void Increment()
{
char[] tmp_buffer = word_buffer;
BUFFER_SIZE += INC;
word_buffer = new char[BUFFER_SIZE];
tmp_buffer.CopyTo(word_buffer, 0);
tmp_buffer = null;
}
// is char vowel
/*
* The following letters are vowels:
* a e i o u y ä ö ü
*/
private bool vowel(char ch)
{
switch (ch)
{
case 'a':
case 'e':
case 'i':
case 'o':
case 'u':
case 'y':
case 'ä':
case 'ö':
case 'ü': return true;
default: return false;
}
}
// R1 and R2 set up in the standard way
private int DefineR(int start)
{
int len = STR_SIZE;
if (start == 0)
start = 1;
if ((start < len) && (start > 0))
{
for (int i = start; i < len; ++i)
{
if ((!vowel(word_buffer[i])) && vowel(word_buffer[(i - 1)]))
return ((i - start) + start + 1);
}
}
return -1;
}
private void SetWord(string strWord)
{
// First, replace ß by ss
string tmp = strWord.Replace("ß", "ss");
// adjust buffer size
while (tmp.Length > BUFFER_SIZE)
Increment();
STR_SIZE = tmp.Length;
// fill in to buffer
tmp.CopyTo(0, word_buffer, 0, STR_SIZE);
}
private void CutEnd(int count)
{
STR_SIZE -= count;
}
private bool EndsWith(string end)
{
if (STR_SIZE > end.Length)
{
int stop = STR_SIZE - end.Length - 1;
int j = end.Length - 1;
for (int i = (STR_SIZE - 1); i > stop; --i)
{
if (word_buffer[i] != end[j])
return false;
--j;
}
return true;
}
return false;
}
private void preprocess()
{
// put u and y between vowels into upper case
for (int i = 1; i < STR_SIZE; ++i)
{
if (word_buffer[i] == 'u')
{
if (vowel(word_buffer[(i - 1)]) && vowel(word_buffer[(i + 1)]))
word_buffer[i] = 'U';
}
else if (word_buffer[i] == 'y')
{
if (vowel(word_buffer[(i - 1)]) && vowel(word_buffer[(i + 1)]))
word_buffer[i] = 'Y';
}
}
/*
* R1 and R2 are first set up in the standard way,
* but then R1 is adjusted
* so that the region before it contains at least 3 letters
*/
R1 = DefineR(0);
R2 = DefineR(R1);
if ((R1 < 3) && (R1 > -1))
R1 = 3;
}
/*
* Search for the longest among the following suffixes,
* (a) e em en ern er es
* (b) s (preceded by a valid s-ending)
* and delete if in R1.
* (Of course the letter of the valid s-ending is not necessarily in R1)
*
* (For example, äckern -> äck, ackers -> acker, armes -> arm)
*/
private void Step1()
{
if (R1 < 0)
return;
// e em en ern er es
if (EndsWith("ern"))
{
if ((STR_SIZE - R1) >= 3)
CutEnd(3);
}
else if (EndsWith("em") || EndsWith("en") || EndsWith("er") || EndsWith("es"))
{
if ((STR_SIZE - R1) >= 2)
CutEnd(2);
}
else if (EndsWith("e"))
{
if ((STR_SIZE - R1) >= 1)
CutEnd(1);
}
// b, d, f, g, h, k, l, m, n, r or t
else if (EndsWith("bs") || EndsWith("ds") || EndsWith("fs") ||
EndsWith("gs") || EndsWith("hs") || EndsWith("ks") ||
EndsWith("ls") || EndsWith("ms") || EndsWith("ns") ||
EndsWith("rs") || EndsWith("ts"))
{
if ((STR_SIZE - R1) >= 1)
CutEnd(1);
}
}
/*
* Search for the longest among the following suffixes,
* (a) en er est
* (b) st (preceded by a valid st-ending, itself preceded by at least 3 letters)
* and delete if in R1.
*
* (For example, derbsten -> derbst by step 1, and derbst -> derb by step 2, since b is a valid st-ending, and is preceded by just 3 letters)
*/
private void Step2()
{
if (R1 < 0)
return;
// en er est
if (EndsWith("est"))
{
if ((STR_SIZE - R1) >= 3)
CutEnd(3);
}
else if (EndsWith("en") || EndsWith("er"))
{
if ((STR_SIZE - R1) >= 2)
CutEnd(2);
}
// b, d, f, g, h, k, l, m, n or t
else if (EndsWith("bst") || EndsWith("dst") || EndsWith("fst") ||
EndsWith("gst") || EndsWith("hst") || EndsWith("kst") ||
EndsWith("lst") || EndsWith("mst") || EndsWith("nst") ||
EndsWith("tst"))
{
// preceded by at least 3 letters
if (STR_SIZE > 5)
{
if ((STR_SIZE - R1) >= 2)
CutEnd(2);
}
}
}
private void Step3()
{
if ((R2 < 0) || (R1 < 0))
return;
/*
* Search for the longest among the following suffixes,
* and perform the action indicated.
* end ung
* delete if in R2
* if preceded by ig, delete if in R2 and not preceded by e
*/
if (EndsWith("end") || EndsWith("ung"))
{
if ((STR_SIZE - R2) >= 3)
CutEnd(3);
if (EndsWith("ig") && (word_buffer[(STR_SIZE - 3)] != 'e'))
{
if ((STR_SIZE - R2) >= 2)
CutEnd(2);
}
}
/*
* ig ik isch
* delete if in R2 and not preceded by e
*/
else if ((EndsWith("ig") || EndsWith("ik")) && (word_buffer[(STR_SIZE - 3)] != 'e'))
{
if ((STR_SIZE - R2) >= 2)
CutEnd(2);
}
else if (EndsWith("isch") && (word_buffer[(STR_SIZE - 5)] != 'e'))
{
if ((STR_SIZE - R2) >= 4)
CutEnd(4);
}
/*
* lich heit
* delete if in R2
* if preceded by er or en, delete if in R1
*/
else if (EndsWith("lich") || EndsWith("heit"))
{
CutEnd(4);
// if preceded by er or en, delete if in R1
if (EndsWith("en") || EndsWith("er"))
{
if ((STR_SIZE - R1) >= 2)
CutEnd(2);
else
STR_SIZE += 4;
}
else
{
STR_SIZE += 4;
if ((STR_SIZE - R2) >= 4)
CutEnd(4);
}
}
/*
* keit
* delete if in R2
* if preceded by lich or ig, delete if in R2
*/
else if (EndsWith("keit"))
{
if ((STR_SIZE - R2) >= 4)
CutEnd(4);
// if preceded by lich or ig, delete if in R2
if (EndsWith("ig"))
{
if ((STR_SIZE - R2) >= 2)
CutEnd(2);
}
else if (EndsWith("lich"))
{
if ((STR_SIZE - R2) >= 4)
CutEnd(4);
}
}
}
// Turn U and Y back into lower case,
// and remove the umlaut accent from a, o and u.
private void Finally()
{
for (int i = 0; i < STR_SIZE; ++i)
{
switch (word_buffer[i])
{
case 'ä':
word_buffer[i] = 'a'; break;
case 'U':
word_buffer[i] = 'u'; break;
case 'ü':
word_buffer[i] = 'u'; break;
case 'Y':
word_buffer[i] = 'y'; break;
case 'ö':
word_buffer[i] = 'o'; break;
default: break;
}
}
}
private string Stem()
{
preprocess();
Step1();
Step2();
Step3();
Finally();
// return stemed word
return new string(word_buffer, 0, STR_SIZE);
}
private int counter = 0;
public string Lex(string word)
{
if (counter > 0)
throw new Exception("German stemmer is not reenterable. Create new instance");
Interlocked.Increment(ref counter);
try
{
Word = word.ToLowerInvariant();
return Stem();
}
finally
{
Interlocked.Decrement(ref counter);
}
}
private string Word
{
get
{
return new string(word_buffer, 0, STR_SIZE);
}
set
{
SetInitState();
SetWord(value);
}
}
}
}