edkuksa/russianmorphology
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- removed deprecated standart filter

Browse Source 
- version 1.5
- lucene version 8.0
- new syntax for TokenStreamComponents
This commit is contained in:
ivansorokin 2019-06-26 23:34:09 +05:00
parent 48b1ceeb3a
commit 3fe076556f
9 changed files with 23 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -5,7 +5,7 @@ Russian and English morphology for Java and [Apache Lucene](http://lucene.apache
### How to use
Build project, by running `mvn clean package`, this will provide you the latest versions of the artifacts - 1.4, add it to your classpath. You could select which version to use - Russian or English.
Build project, by running `mvn clean package`, this will provide you the latest versions of the artifacts - 1.5, add it to your classpath. You could select which version to use - Russian or English.
Now you can create a Lucene Analyzer:
@ -39,7 +39,7 @@ You can use the LuceneMorphology as morphology filter in a Solr _schema.xml_ usi
</fieldType>
```
Just add _morphology-1.4.jar_ in your Solr lib-directories
Just add _morphology-1.5.jar_ in your Solr lib-directories
### Restrictions

8
dictionary-reader/pom.xml
View File

@ -3,27 +3,27 @@
<parent>
<artifactId>morphology</artifactId>
<groupId>org.apache.lucene.morphology</groupId>
<version>1.4</version>
<version>1.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>dictionary-reader</artifactId>
<name>dictionary-reader</name>
<version>1.4</version>
<version>1.5</version>
<url>http://maven.apache.org</url>
<dependencies>
<dependency>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>russian</artifactId>
<version>1.4</version>
<version>1.5</version>
</dependency>
<dependency>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>english</artifactId>
<version>1.4</version>
<version>1.5</version>
</dependency>
</dependencies>

4
dictionary-reader/src/test/java/org/apache/lucene/morphology/AnalyzersTest.java
View File

@ -21,7 +21,6 @@ import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@ -149,10 +148,9 @@ public class AnalyzersTest extends BaseTokenStreamTestCase {
@Override
protected TokenStreamComponents createComponents(String s) {
StandardTokenizer src = new StandardTokenizer();
TokenFilter filter = new StandardFilter(src);
CharArraySet dontStem = new CharArraySet(1, false);
dontStem.add("Tests");
filter = new SetKeywordMarkerFilter(filter, dontStem);
TokenFilter filter = new SetKeywordMarkerFilter(src, dontStem);
filter = new LowerCaseFilter(filter);
try {
filter = new MorphologyFilter(filter, new EnglishLuceneMorphology());

6
english/pom.xml
View File

@ -3,20 +3,20 @@
<parent>
<artifactId>morphology</artifactId>
<groupId>org.apache.lucene.morphology</groupId>
<version>1.4</version>
<version>1.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>english</artifactId>
<name>english</name>
<version>1.4</version>
<version>1.5</version>
<url>http://maven.apache.org</url>
<dependencies>
<dependency>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>morph</artifactId>
<version>1.4</version>
<version>1.5</version>
</dependency>
</dependencies>

4
morph/pom.xml
View File

@ -3,13 +3,13 @@
<parent>
<artifactId>morphology</artifactId>
<groupId>org.apache.lucene.morphology</groupId>
<version>1.4</version>
<version>1.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>morph</artifactId>
<name>morph</name>
<version>1.4</version>
<version>1.5</version>
<url>http://maven.apache.org</url>
</project>

16
morph/src/main/java/org/apache/lucene/morphology/analyzer/MorphologyAnalyzer.java
View File

@ -21,7 +21,6 @@ import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.payloads.PayloadEncoder;
import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.morphology.LetterDecoderEncoder;
import org.apache.lucene.morphology.LuceneMorphology;
@ -29,7 +28,7 @@ import org.apache.lucene.util.BytesRef;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
public class MorphologyAnalyzer extends Analyzer {
private LuceneMorphology luceneMorph;
@ -70,17 +69,10 @@ public class MorphologyAnalyzer extends Analyzer {
return new BytesRef(bytes, 0, bytes.length);
}
};
TokenFilter filter = new StandardFilter(src);
filter = new LowerCaseFilter(filter);
TokenFilter filter = new LowerCaseFilter(src);
filter = new MorphologyFilter(filter, luceneMorph);
return new TokenStreamComponents(src, filter) {
@Override
protected void setReader(final Reader reader) {
super.setReader(reader);
return new TokenStreamComponents(r -> src.setReader(r), filter);
}
};
}
}

6
pom.xml
View File

@ -4,7 +4,7 @@
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>morphology</artifactId>
<packaging>pom</packaging>
<version>1.4</version>
<version>1.5</version>
<name>morphology</name>
<url>http://maven.apache.org</url>
@ -16,8 +16,8 @@
</scm>
<properties>
<lucene.version>7.2.1</lucene.version>
<morphology.version>1.4</morphology.version>
<lucene.version>8.0.0</lucene.version>
<morphology.version>1.5</morphology.version>
<junit.version>4.12</junit.version>
</properties>

6
russian/pom.xml
View File

@ -3,13 +3,13 @@
<parent>
<artifactId>morphology</artifactId>
<groupId>org.apache.lucene.morphology</groupId>
<version>1.4</version>
<version>1.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>russian</artifactId>
<name>russian</name>
<version>1.4</version>
<version>1.5</version>
<url>http://maven.apache.org</url>
<dependencies>
@ -17,7 +17,7 @@
<dependency>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>morph</artifactId>
<version>1.4</version>
<version>1.5</version>
</dependency>
<dependency>

2
solr-morphology-analysis/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>morphology</artifactId>
<groupId>org.apache.lucene.morphology</groupId>
<version>1.4</version>
<version>1.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>