edkuksa/russianmorphology
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working on morph ambiguously resolver

Browse Source
This commit is contained in:
Alexander Kuznetsov 2015-06-22 22:41:21 +03:00
parent 6ca2b27781
commit 3b2e48821a
7 changed files with 201 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
context/pom.xml Normal file
View File

@ -0,0 +1,23 @@
<?xml version="1.0"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>morphology</artifactId>
<version>1.2-SNAPSHOT</version>
</parent>
<groupId>org.apache.lucene.morphology</groupId>
<artifactId>context</artifactId>
<version>1.0-SNAPSHOT</version>
<name>context</name>
<url>http://maven.apache.org</url>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

7
context/src/main/java/org/apache/lucene/morphology/context/CalculateContextItem.java Normal file
View File

@ -0,0 +1,7 @@
package org.apache.lucene.morphology.context;
/**
* Created by alexander on 16.06.15.
*/
public class CalculateContextItem {
}

29
context/src/main/java/org/apache/lucene/morphology/context/ContextItem.java Normal file
View File

@ -0,0 +1,29 @@
package org.apache.lucene.morphology.context;
/**
* Created by alexander on 16.06.15.
*/
public class ContextItem {
String[][] morphInfo;
long count = 0;
public ContextItem(String[][] morphInfo) {
this.morphInfo = morphInfo;
}
public String[][] getMorphInfo() {
return morphInfo;
}
public void setMorphInfo(String[][] morphInfo) {
this.morphInfo = morphInfo;
}
public long getCount() {
return count;
}
public void setCount(long count) {
this.count = count;
}
}

25
context/src/main/java/org/apache/lucene/morphology/context/ContextStats.java Normal file
View File

@ -0,0 +1,25 @@
package org.apache.lucene.morphology.context;
/**
* Created by alexander on 16.06.15.
*/
public class ContextStats {
String[] morphInfo;
double prob;
public String[] getMorphInfo() {
return morphInfo;
}
public void setMorphInfo(String[] morphInfo) {
this.morphInfo = morphInfo;
}
public double getProb() {
return prob;
}
public void setProb(double prob) {
this.prob = prob;
}
}

9
context/src/main/java/org/apache/lucene/morphology/context/ProbClalucator.java Normal file
View File

@ -0,0 +1,9 @@
package org.apache.lucene.morphology.context;
/**
* Created by alexander on 16.06.15.
*/
public class ProbClalucator {
}

104
context/src/main/java/org/apache/lucene/morphology/context/SimpleTokenizer.java Normal file
View File

@ -0,0 +1,104 @@
package org.apache.lucene.morphology.context;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util.AttributeFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Set;
/**
* Created by alexander on 16.06.15.
*/
public class SimpleTokenizer extends Tokenizer {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class);
LinkedList<String> terms;
public final static Set<Character> SEPARATION_LETTERS = new HashSet<>(Arrays.asList(' ', '(', ')', ',', '|', '\t',
'\n', '"', ':', '!', '?', ',', ';', '•'));
public final static Set<Character> MEANING_CHARS = new HashSet<>(Arrays.asList('(', ')', ',', '|',
'"', ':', '!', '?', ',', ';', '•', '.'));
public SimpleTokenizer() {
}
public SimpleTokenizer(AttributeFactory factory) {
super(factory);
}
@Override
final public boolean incrementToken() throws IOException {
if (terms == null) {
createTeams();
}
if (terms.size() > 0) {
String str = terms.poll();
termAtt.setEmpty();
termAtt.append(str);
posAtt.setPositionIncrement(1);
return true;
}
return false;
}
private void createTeams() throws IOException {
terms = new LinkedList<>();
BufferedReader br = new BufferedReader(input);
StringBuilder sb = new StringBuilder();
String s = "";
while ((s = br.readLine()) != null) {
sb.append(s).append(" ");
}
s = sb.toString();
CharTermAttributeImpl currentTerm = new CharTermAttributeImpl();
for (int i = 0; i < s.length(); i++) {
if (checkIsCharSepartor(s, i)) {
if (checkIsCharHasMeaning(s, i)) {
terms.add(s.substring(i, i + 1));
}
String term = currentTerm.toString();
currentTerm.clear();
if (term.length() > 0) {
terms.add(term);
}
} else {
currentTerm.append(s.charAt(i));
}
}
}
private boolean checkIsCharHasMeaning(String s, int i) {
return MEANING_CHARS.contains(s.charAt(i));
}
private boolean checkIsCharSepartor(String s, int i) {
char c = s.charAt(i);
if (SEPARATION_LETTERS.contains(c)) {
return true;
}
if ('.' == c
&& s.length() > i + 1
&& SEPARATION_LETTERS.contains(s.charAt(i + 1))) {
return true;
}
return false;
}
@Override
public void reset() throws IOException {
this.terms = null;
super.reset();
}
}

5
pom.xml
View File

@ -94,8 +94,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.5</source>
<target>1.5</target>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin> <!-- usage: http://code.google.com/p/maven-license-plugin/wiki/HowTo -->
@ -160,5 +160,6 @@
<module>dictionary-reader</module>
<module>russian</module>
<module>english</module>
<module>context</module>
</modules>
</project>