edkuksa/russianmorphology
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding support of lucene 3.5 and start working english stemmer

Browse Source 
git-svn-id: https://russianmorphology.googlecode.com/svn/trunk@123 d817d54c-26ab-11de-abc9-2f7d1455ff7a
This commit is contained in:
Alexander.A.Kuznetsov@gmail.com
2011-11-28 16:44:15 +00:00
parent d46651f2ba
commit 466de768ca
6 changed files with 75 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
morph/src/main/java/org/apache/lucene/morphology/analyzer/MorphologyAnalyzer.java
View File

@ -44,10 +44,15 @@ public class MorphologyAnalyzer extends Analyzer {
luceneMorph = new LuceneMorphology(inputStream, letterDecoderEncoder);
}
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new StandardTokenizer(Version.LUCENE_30, reader);
result = new StandardFilter(result);
result = new LowerCaseFilter(result);
final public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new StandardTokenizer(Version.LUCENE_35, reader);
result = new StandardFilter(Version.LUCENE_35,result);
result = new LowerCaseFilter(Version.LUCENE_35,result);
return new MorphologyFilter(result, luceneMorph);
}
@Override
final public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
return super.reusableTokenStream(fieldName, reader);
}
}

14
morph/src/main/java/org/apache/lucene/morphology/analyzer/MorphologyFilter.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.morphology.analyzer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.morphology.LuceneMorphology;
@ -28,30 +29,31 @@ import java.util.Iterator;
public class MorphologyFilter extends TokenFilter {
private LuceneMorphology luceneMorph;
private Iterator<String> iterator;
private TermAttribute termAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
public MorphologyFilter(TokenStream tokenStream, LuceneMorphology luceneMorph) {
super(tokenStream);
this.luceneMorph = luceneMorph;
termAtt = addAttribute(TermAttribute.class);
}
public boolean incrementToken() throws IOException {
final public boolean incrementToken() throws IOException {
while (iterator == null || !iterator.hasNext()) {
boolean b = input.incrementToken();
if (!b) {
return false;
}
String s = termAtt.term();
String s = new String(termAtt.buffer(),0,termAtt.length());
if (luceneMorph.checkString(s)) {
iterator = luceneMorph.getNormalForms(termAtt.term()).iterator();
iterator = luceneMorph.getNormalForms(s).iterator();
} else {
return true;
}
}
String s = iterator.next();
termAtt.setTermBuffer(s);
termAtt.setEmpty();
termAtt.append(s);
return true;
}