making some clear up
git-svn-id: https://russianmorphology.googlecode.com/svn/trunk@63 d817d54c-26ab-11de-abc9-2f7d1455ff7a
This commit is contained in:
alexander.a.kuznetsov
parent
8c833132a8
commit
c858d2560a
@ -1,121 +0,0 @@
|
|||||||
// טלוםא
|
|
||||||
במ C <20> לנ,טל<D798>,וה,טל
|
|
||||||
בן C <20> לנ,טל<D798>,וה,נה
|
|
||||||
בנ C <20> לנ,טל<D798>,וה,הע
|
|
||||||
בס C <20> לנ,טל<D798>,וה,גם
|
|
||||||
בע C <20> לנ,טל<D798>,וה,עג
|
|
||||||
בף C <20> לנ,טל<D798>,וה,ןנ
|
|
||||||
ב<EFBFBD> C <20> לנ,טל<D798>,וה,חג,נאחד
|
|
||||||
בפ C <20> לנ,טל<D798>,לם,טל
|
|
||||||
בץ C <20> לנ,טל<D798>,לם,נה
|
|
||||||
בצ C <20> לנ,טל<D798>,לם,הע
|
|
||||||
בק C <20> לנ,טל<D798>,לם,גם
|
|
||||||
בר C <20> לנ,טל<D798>,לם,עג
|
|
||||||
בש C <20> לנ,טל<D798>,לם,ןנ
|
|
||||||
ב<EFBFBD> I <20> לנ,טל<D798>,0
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,0
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,וה,טל
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,וה,נה
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,וה,הע
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,וה,גם
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,וה,עג
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,וה,ןנ
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,וה,חג,נאחד
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,לם,טל
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,לם,נה
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,לם,הע
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,לם,גם
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,לם,עג
|
|
||||||
ג<EFBFBD> E <20> לנ-זנ,טל<D798>,לם,ןנ
|
|
||||||
המ I <20> זנ,טל<D798>,וה,טל
|
|
||||||
הן I <20> זנ,טל<D798>,וה,נה
|
|
||||||
הנ I <20> זנ,טל<D798>,וה,הע
|
|
||||||
הס I <20> זנ,טל<D798>,וה,גם
|
|
||||||
הע I <20> זנ,טל<D798>,וה,עג
|
|
||||||
הף I <20> זנ,טל<D798>,וה,ןנ
|
|
||||||
ה<EFBFBD> I <20> זנ,טל<D798>,וה,חג,נאחד
|
|
||||||
הפ I <20> זנ,טל<D798>,לם,טל
|
|
||||||
הץ I <20> זנ,טל<D798>,לם,נה
|
|
||||||
הצ I <20> זנ,טל<D798>,לם,הע
|
|
||||||
הק I <20> זנ,טל<D798>,לם,גם
|
|
||||||
הר I <20> זנ,טל<D798>,לם,עג
|
|
||||||
הש I <20> זנ,טל<D798>,לם,ןנ
|
|
||||||
ה<EFBFBD> I <20> זנ,טל<D798>,0
|
|
||||||
// לףזסךטו מעקוסעגא
|
|
||||||
<EFBFBD>א Q <20> לנ,מעק,וה,טל,
|
|
||||||
<EFBFBD>ב Q <20> לנ,מעק,וה,נה,
|
|
||||||
<EFBFBD>ג Q <20> לנ,מעק,וה,הע,
|
|
||||||
<EFBFBD>ד Q <20> לנ,מעק,וה,גם,
|
|
||||||
<EFBFBD>ה Q <20> לנ,מעק,וה,עג,
|
|
||||||
<EFBFBD>ו Q <20> לנ,מעק,וה,ןנ,
|
|
||||||
<EFBFBD>ם Q <20> לנ,מעק,לם,טל,
|
|
||||||
<EFBFBD>מ Q <20> לנ,מעק,לם,נה,
|
|
||||||
<EFBFBD>ן Q <20> לנ,מעק,לם,הע,
|
|
||||||
<EFBFBD>נ Q <20> לנ,מעק,לם,גם,
|
|
||||||
<EFBFBD>ס Q <20> לנ,מעק,לם,עג,
|
|
||||||
<EFBFBD>ע Q <20> לנ,מעק,לם,ןנ,
|
|
||||||
// זוםסךטו מעקוסעגא
|
|
||||||
<EFBFBD>ז Q <20> זנ,מעק,וה,טל,
|
|
||||||
<EFBFBD>ח Q <20> זנ,מעק,וה,נה,
|
|
||||||
<EFBFBD>ט Q <20> זנ,מעק,וה,הע,
|
|
||||||
<EFBFBD>ך Q <20> זנ,מעק,וה,גם,
|
|
||||||
<EFBFBD>כ Q <20> זנ,מעק,וה,עג,
|
|
||||||
<EFBFBD>ל Q <20> זנ,מעק,וה,ןנ,
|
|
||||||
<EFBFBD>ף Q <20> זנ,מעק,לם,טל,
|
|
||||||
<EFBFBD>פ Q <20> זנ,מעק,לם,נה,
|
|
||||||
<EFBFBD>ץ Q <20> זנ,מעק,לם,הע,
|
|
||||||
<EFBFBD>צ Q <20> זנ,מעק,לם,גם,
|
|
||||||
<EFBFBD>ק Q <20> זנ,מעק,לם,עג,
|
|
||||||
<EFBFBD>ר Q <20> זנ,מעק,לם,ןנ,
|
|
||||||
// לףזסךטו מעקוסעגא
|
|
||||||
<EFBFBD>א Q <20> לנ,מעק,וה,טל,
|
|
||||||
<EFBFBD>ב Q <20> לנ,מעק,וה,נה,
|
|
||||||
<EFBFBD>ג Q <20> לנ,מעק,וה,הע,
|
|
||||||
<EFBFBD>ד Q <20> לנ,מעק,וה,גם,
|
|
||||||
<EFBFBD>ה Q <20> לנ,מעק,וה,עג,
|
|
||||||
<EFBFBD>ו Q <20> לנ,מעק,וה,ןנ,
|
|
||||||
<EFBFBD>ם Q <20> לנ,מעק,לם,טל,
|
|
||||||
<EFBFBD>מ Q <20> לנ,מעק,לם,נה,
|
|
||||||
<EFBFBD>ן Q <20> לנ,מעק,לם,הע,
|
|
||||||
<EFBFBD>נ Q <20> לנ,מעק,לם,גם,
|
|
||||||
<EFBFBD>ס Q <20> לנ,מעק,לם,עג,
|
|
||||||
<EFBFBD>ע Q <20> לנ,מעק,לם,ןנ,
|
|
||||||
// זוםסךטו מעקוסעגא
|
|
||||||
<EFBFBD>ז Q <20> זנ,מעק,וה,טל,
|
|
||||||
<EFBFBD>ח Q <20> זנ,מעק,וה,נה,
|
|
||||||
<EFBFBD>ט Q <20> זנ,מעק,וה,הע,
|
|
||||||
<EFBFBD>ך Q <20> זנ,מעק,וה,גם,
|
|
||||||
<EFBFBD>כ Q <20> זנ,מעק,וה,עג,
|
|
||||||
<EFBFBD>ל Q <20> זנ,מעק,וה,ןנ,
|
|
||||||
<EFBFBD>ף Q <20> זנ,מעק,לם,טל,
|
|
||||||
<EFBFBD>פ Q <20> זנ,מעק,לם,נה,
|
|
||||||
<EFBFBD>ץ Q <20> זנ,מעק,לם,הע,
|
|
||||||
<EFBFBD>צ Q <20> זנ,מעק,לם,גם,
|
|
||||||
<EFBFBD>ק Q <20> זנ,מעק,לם,עג,
|
|
||||||
<EFBFBD>ר Q <20> זנ,מעק,לם,ןנ,
|
|
||||||
// לףזסךטו מעקוסעגא (נאחד.)
|
|
||||||
<EFBFBD>א Q <20> לנ,מעק,נאחד,וה,טל,
|
|
||||||
<EFBFBD>ב Q <20> לנ,מעק,נאחד,וה,נה,
|
|
||||||
<EFBFBD>ג Q <20> לנ,מעק,נאחד,וה,הע,
|
|
||||||
<EFBFBD>ד Q <20> לנ,מעק,נאחד,וה,גם,
|
|
||||||
<EFBFBD>ה Q <20> לנ,מעק,נאחד,וה,עג,
|
|
||||||
<EFBFBD>ו Q <20> לנ,מעק,נאחד,וה,ןנ,
|
|
||||||
<EFBFBD>ם Q <20> לנ,מעק,נאחד,לם,טל,
|
|
||||||
<EFBFBD>מ Q <20> לנ,מעק,נאחד,לם,נה,
|
|
||||||
<EFBFBD>ן Q <20> לנ,מעק,נאחד,לם,הע,
|
|
||||||
<EFBFBD>נ Q <20> לנ,מעק,נאחד,לם,גם,
|
|
||||||
<EFBFBD>ס Q <20> לנ,מעק,נאחד,לם,עג,
|
|
||||||
<EFBFBD>ע Q <20> לנ,מעק,נאחד,לם,ןנ,
|
|
||||||
// זוםסךטו מעקוסעגא (נאחד.)
|
|
||||||
<EFBFBD>ז Q <20> זנ,מעק,נאחד,וה,טל,
|
|
||||||
<EFBFBD>ח Q <20> זנ,מעק,נאחד,וה,נה,
|
|
||||||
<EFBFBD>ט Q <20> זנ,מעק,נאחד,וה,הע,
|
|
||||||
<EFBFBD>ך Q <20> זנ,מעק,נאחד,וה,גם,
|
|
||||||
<EFBFBD>כ Q <20> זנ,מעק,נאחד,וה,עג,
|
|
||||||
<EFBFBD>ל Q <20> זנ,מעק,נאחד,וה,ןנ,
|
|
||||||
<EFBFBD>ף Q <20> זנ,מעק,נאחד,לם,טל,
|
|
||||||
<EFBFBD>פ Q <20> זנ,מעק,נאחד,לם,נה,
|
|
||||||
<EFBFBD>ץ Q <20> זנ,מעק,נאחד,לם,הע,
|
|
||||||
<EFBFBD>צ Q <20> זנ,מעק,נאחד,לם,גם,
|
|
||||||
<EFBFBD>ק Q <20> זנ,מעק,נאחד,לם,עג,
|
|
||||||
<EFBFBD>ר Q <20> זנ,מעק,נאחד,לם,ןנ,
|
|
||||||
32459
data/lemma.num
32459
data/lemma.num
File diff suppressed because it is too large
Load Diff
@ -1,56 +0,0 @@
|
|||||||
/**
|
|
||||||
* Copyright 2009 Alexander Kuznetsov
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.apache.lucene.morphology.dictionary;
|
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
|
||||||
import java.io.FileInputStream;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStreamReader;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
|
|
||||||
public class FrequentyReader {
|
|
||||||
private String fileName;
|
|
||||||
private String fileEncoding = "windows-1251";
|
|
||||||
|
|
||||||
public FrequentyReader(String fileName) {
|
|
||||||
this.fileName = fileName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public FrequentyReader(String fileName, String fileEncoding) {
|
|
||||||
this.fileName = fileName;
|
|
||||||
this.fileEncoding = fileEncoding;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public Map<String, Double> read() throws IOException {
|
|
||||||
Map<String, Double> result = new HashMap<String, Double>();
|
|
||||||
|
|
||||||
BufferedReader bufferedReader = new BufferedReader(
|
|
||||||
new InputStreamReader(
|
|
||||||
new FileInputStream(fileName), fileEncoding));
|
|
||||||
String s = bufferedReader.readLine();
|
|
||||||
while (s != null) {
|
|
||||||
String[] strings = s.split(" ");
|
|
||||||
Double value = Double.valueOf(strings[1]);
|
|
||||||
result.put(strings[2], value);
|
|
||||||
s = bufferedReader.readLine();
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,54 +0,0 @@
|
|||||||
/**
|
|
||||||
* Copyright 2009 Alexander Kuznetsov
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.apache.lucene.morphology.dictionary;
|
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
|
||||||
import java.io.FileInputStream;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStreamReader;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
|
|
||||||
public class IgnoredFormReader {
|
|
||||||
private String fileName;
|
|
||||||
private String fileEncoding = "windows-1251";
|
|
||||||
|
|
||||||
public IgnoredFormReader(String fileName) {
|
|
||||||
this.fileName = fileName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IgnoredFormReader(String fileName, String fileEncoding) {
|
|
||||||
this.fileName = fileName;
|
|
||||||
this.fileEncoding = fileEncoding;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Set<String> getIngnoredFroms() throws IOException {
|
|
||||||
BufferedReader bufferedReader = new BufferedReader(
|
|
||||||
new InputStreamReader(
|
|
||||||
new FileInputStream(fileName), fileEncoding));
|
|
||||||
String s = bufferedReader.readLine();
|
|
||||||
HashSet<String> result = new HashSet<String>();
|
|
||||||
while (s != null) {
|
|
||||||
if (!s.startsWith("//")) {
|
|
||||||
result.add(s.trim().split(" ")[0]);
|
|
||||||
}
|
|
||||||
s = bufferedReader.readLine();
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -25,7 +25,7 @@ import java.io.IOException;
|
|||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
|
|
||||||
//todo made refactoring thi8s class
|
//todo made refactoring this class
|
||||||
public class StatiticsCollector implements WordProccessor {
|
public class StatiticsCollector implements WordProccessor {
|
||||||
private TreeMap<String, Set<Heuristic>> inversIndex = new TreeMap<String, Set<Heuristic>>();
|
private TreeMap<String, Set<Heuristic>> inversIndex = new TreeMap<String, Set<Heuristic>>();
|
||||||
private Map<Set<Heuristic>, Integer> ruleInverIndex = new HashMap<Set<Heuristic>, Integer>();
|
private Map<Set<Heuristic>, Integer> ruleInverIndex = new HashMap<Set<Heuristic>, Integer>();
|
||||||
|
|||||||
@ -27,8 +27,6 @@ import java.util.HashSet;
|
|||||||
|
|
||||||
public class EnglishHeuristicBuilder {
|
public class EnglishHeuristicBuilder {
|
||||||
public static void main(String[] args) throws IOException {
|
public static void main(String[] args) throws IOException {
|
||||||
//IgnoredFormReader formReader = new IgnoredFormReader("data/igoredFrom.txt");
|
|
||||||
//Set<String> form = formReader.getIngnoredFroms();
|
|
||||||
|
|
||||||
GrammaReader grammaInfo = new GrammaReader("dictonary/Dicts/Morph/egramtab.tab");
|
GrammaReader grammaInfo = new GrammaReader("dictonary/Dicts/Morph/egramtab.tab");
|
||||||
DictonaryReader dictonaryReader = new DictonaryReader("dictonary/Dicts/SrcMorph/EngSrc/morphs.mrd", new HashSet<String>());
|
DictonaryReader dictonaryReader = new DictonaryReader("dictonary/Dicts/SrcMorph/EngSrc/morphs.mrd", new HashSet<String>());
|
||||||
|
|||||||
@ -27,9 +27,6 @@ import java.util.HashSet;
|
|||||||
|
|
||||||
public class RussianHeuristicBuilder {
|
public class RussianHeuristicBuilder {
|
||||||
public static void main(String[] args) throws IOException {
|
public static void main(String[] args) throws IOException {
|
||||||
//IgnoredFormReader formReader = new IgnoredFormReader("data/igoredFrom.txt");
|
|
||||||
//Set<String> form = formReader.getIngnoredFroms();
|
|
||||||
|
|
||||||
GrammaReader grammaInfo = new GrammaReader("dictonary/Dicts/Morph/rgramtab.tab");
|
GrammaReader grammaInfo = new GrammaReader("dictonary/Dicts/Morph/rgramtab.tab");
|
||||||
DictonaryReader dictonaryReader = new DictonaryReader("dictonary/Dicts/SrcMorph/RusSrc/morphs.mrd", new HashSet<String>());
|
DictonaryReader dictonaryReader = new DictonaryReader("dictonary/Dicts/SrcMorph/RusSrc/morphs.mrd", new HashSet<String>());
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user