edkuksa/russianmorphology
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding english version

Browse Source 
git-svn-id: https://russianmorphology.googlecode.com/svn/trunk@57 d817d54c-26ab-11de-abc9-2f7d1455ff7a
This commit is contained in:
alexander.a.kuznetsov
2009-10-15 18:19:31 +00:00
parent 1b8ee03cc6
commit a1e39d750f
19 changed files with 105555 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
dictionary-reader/pom.xml
View File

@ -19,6 +19,13 @@
<version>0.7-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.lucene.morpholgy</groupId>
<artifactId>english</artifactId>
<version>0.7-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.lucene.morpholgy</groupId>
<artifactId>morph</artifactId>

6
dictionary-reader/src/main/java/org/apache/lucene/morpholgy/dictionary/StatiticsCollector.java
View File

@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.*;
//todo made refactoring thi8s class
public class StatiticsCollector implements WordProccessor {
private TreeMap<String, Set<Heuristic>> inversIndex = new TreeMap<String, Set<Heuristic>>();
private Map<Set<Heuristic>, Integer> ruleInverIndex = new HashMap<Set<Heuristic>, Integer>();
@ -43,8 +44,10 @@ public class StatiticsCollector implements WordProccessor {
String normalStringMorph = wordCard.getWordsFroms().get(0).getCode();
String word = wordCard.getBase() + wordCard.getCanonicalSuffix();
if (word.contains("-")) return;
if (!decoderEncoder.checkString(word)) return;
for (FlexiaModel fm : wordCard.getWordsFroms()) {
if (!decoderEncoder.checkString(fm.create(wordCard.getBase()))) continue;
Heuristic heuristic = createEvristic(wordCard.getBase(), wordCard.getCanonicalSuffix(), fm, normalStringMorph);
String form = revertWord(fm.create(wordCard.getBase()));
Set<Heuristic> suffixHeuristics = inversIndex.get(form);
@ -109,7 +112,8 @@ public class StatiticsCollector implements WordProccessor {
for (String key : inversIndex.keySet()) {
Set<Heuristic> currentSet = inversIndex.get(key);
if (!currentSet.equals(prevSet)) {
ints[count] = decoderEncoder.encodeToArray(key);
int[] word = decoderEncoder.encodeToArray(key);
ints[count] = word;
rulesId[count] = (short) ruleInverIndex.get(currentSet).intValue();
count++;
prevSet = currentSet;

42
dictionary-reader/src/main/java/org/apache/lucene/morpholgy/generator/EnglishHeuristicBuilder.java Normal file
View File

@ -0,0 +1,42 @@
/**
* Copyright 2009 Alexander Kuznetsov
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.morpholgy.generator;
import org.apache.lucene.morpholgy.dictionary.DictonaryReader;
import org.apache.lucene.morpholgy.dictionary.GrammaReader;
import org.apache.lucene.morpholgy.dictionary.StatiticsCollector;
import org.apache.lucene.morpholgy.english.EnglishLetterDecoderEncoder;
import java.io.IOException;
import java.util.HashSet;
public class EnglishHeuristicBuilder {
public static void main(String[] args) throws IOException {
//IgnoredFormReader formReader = new IgnoredFormReader("data/igoredFrom.txt");
//Set<String> form = formReader.getIngnoredFroms();
GrammaReader grammaInfo = new GrammaReader("dictonary/Dicts/Morph/egramtab.tab");
DictonaryReader dictonaryReader = new DictonaryReader("dictonary/Dicts/SrcMorph/EngSrc/morphs.mrd", new HashSet<String>());
EnglishLetterDecoderEncoder decoderEncoder = new EnglishLetterDecoderEncoder();
StatiticsCollector statiticsCollector = new StatiticsCollector(grammaInfo, decoderEncoder);
dictonaryReader.proccess(statiticsCollector);
statiticsCollector.saveHeuristic("english/src/main/resources/org/apache/lucene/morphology/english/morph.info");
}
}