edkuksa/russianmorphology
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

working on new version

Browse Source 
git-svn-id: https://russianmorphology.googlecode.com/svn/trunk@51 d817d54c-26ab-11de-abc9-2f7d1455ff7a
This commit is contained in:
alexander.a.kuznetsov
2009-10-03 10:58:54 +00:00
parent ca1a9be6b7
commit fed6cd480a
10 changed files with 69 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
morph/src/main/java/org/apache/lucene/morphology/LuceneMorph.java
View File

@ -23,10 +23,9 @@ import java.util.List;
public class LuceneMorph extends Morph {
LetterDecoderEncoder decoderEncoder;
public LuceneMorph(String fileName) throws IOException {
super(fileName);
public LuceneMorph(String fileName,LetterDecoderEncoder decoderEncoder) throws IOException {
super(fileName,decoderEncoder);
}
@Override

5
morph/src/main/java/org/apache/lucene/morphology/Morph.java
View File

@ -29,11 +29,12 @@ public class Morph {
protected short[] rulesId;
protected Heuristic[][] rules;
protected String[] grammaInfo;
LetterDecoderEncoder decoderEncoder;
protected LetterDecoderEncoder decoderEncoder;
public Morph(String fileName) throws IOException {
public Morph(String fileName,LetterDecoderEncoder decoderEncoder) throws IOException {
readFromFile(fileName);
this.decoderEncoder = decoderEncoder;
}
public Morph(int[][] separators, short[] rulesId, Heuristic[][] rules, String[] grammaInfo) {

9
morph/src/main/java/org/apache/lucene/morphology/analayzer/RussianMorphlogyAnalayzer.java → morph/src/main/java/org/apache/lucene/morphology/analayzer/MorphlogyAnalayzer.java
View File

@ -22,21 +22,22 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.morphology.LuceneMorph;
import org.apache.lucene.morphology.LetterDecoderEncoder;
import java.io.IOException;
import java.io.Reader;
public class RussianMorphlogyAnalayzer extends Analyzer {
public class MorphlogyAnalayzer extends Analyzer {
private LuceneMorph luceneMorph;
public RussianMorphlogyAnalayzer() throws IOException {
luceneMorph = new LuceneMorph("sep.txt");
public MorphlogyAnalayzer(String pathToMorph,LetterDecoderEncoder letterDecoderEncoder) throws IOException {
luceneMorph = new LuceneMorph("sep.txt",letterDecoderEncoder);
}
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new StandardTokenizer(reader);
result = new StandardFilter(result);
result = new LowerCaseFilter(result);
return new RussianMorphlogyFilter(result, luceneMorph);
return new MorphlogyFilter(result, luceneMorph);
}
}

5
morph/src/main/java/org/apache/lucene/morphology/analayzer/RussianMorphlogyFilter.java → morph/src/main/java/org/apache/lucene/morphology/analayzer/MorphlogyFilter.java
View File

@ -26,10 +26,10 @@ import java.util.ArrayList;
import java.util.List;
public class RussianMorphlogyFilter extends TokenFilter {
public class MorphlogyFilter extends TokenFilter {
private LuceneMorph luceneMorph;
public RussianMorphlogyFilter(TokenStream tokenStream, LuceneMorph luceneMorph) {
public MorphlogyFilter(TokenStream tokenStream, LuceneMorph luceneMorph) {
super(tokenStream);
this.luceneMorph = luceneMorph;
}
@ -54,6 +54,7 @@ public class RussianMorphlogyFilter extends TokenFilter {
Token nextToken = input.next(reusableToken);
if (nextToken == null) return null; // EOS; iterator exhausted
Character testC = nextToken.term().charAt(0);
//todo check here for decoder endocoder
if (Character.UnicodeBlock.of(testC) != Character.UnicodeBlock.CYRILLIC) {
return nextToken;
}