Adding context stats

context/pom.xml

@@ -1,23 +1,36 @@
 <?xml version="1.0"?>
-<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+         xmlns="http://maven.apache.org/POM/4.0.0"
-  <modelVersion>4.0.0</modelVersion>
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-  <parent>
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.lucene.morphology</groupId>
+        <artifactId>morphology</artifactId>
+        <version>1.2-SNAPSHOT</version>
+    </parent>
     <groupId>org.apache.lucene.morphology</groupId>
-    <artifactId>morphology</artifactId>
+    <artifactId>context</artifactId>
-    <version>1.2-SNAPSHOT</version>
+    <version>1.0-SNAPSHOT</version>
-  </parent>
+    <name>context</name>
-  <groupId>org.apache.lucene.morphology</groupId>
+    <url>http://maven.apache.org</url>
-  <artifactId>context</artifactId>
+    <dependencies>
-  <version>1.0-SNAPSHOT</version>
+        <dependency>
-  <name>context</name>
+            <groupId>junit</groupId>
-  <url>http://maven.apache.org</url>
+            <artifactId>junit</artifactId>
-  <dependencies>
+            <version>4.8.2</version>
-    <dependency>
+            <scope>test</scope>
-      <groupId>junit</groupId>
+        </dependency>
-      <artifactId>junit</artifactId>
+        <dependency>
-      <version>3.8.1</version>
+            <groupId>org.apache.lucene.morphology</groupId>
-      <scope>test</scope>
+            <artifactId>russian</artifactId>
-    </dependency>
+            <version>1.2-SNAPSHOT</version>
-  </dependencies>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.lucene.morphology</groupId>
+            <artifactId>english</artifactId>
+            <version>1.2-SNAPSHOT</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 </project>

context/src/main/java/org/apache/lucene/morphology/context/CalculateContextItem.java

@@ -1,7 +1,52 @@
+/**
+ * Copyright 2015 Alexander Kuznetsov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.lucene.morphology.context;
 
-/**
+import org.apache.lucene.analysis.Analyzer;
- * Created by alexander on 16.06.15.
+import org.apache.lucene.analysis.TokenStream;
- */
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
 public class CalculateContextItem {
+
+    public List<ContextItem> createContextItems(String text) throws IOException {
+        Analyzer statAnalyzer = new StatAnalyzer();
+        InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream("принеси мне вина на новый год? - и что я жду тебя, где вино".getBytes()), "UTF-8");
+
+
+//        new RussianMorphology();
+
+        TokenStream tokenStream = statAnalyzer.tokenStream(null, reader);
+        tokenStream.reset();
+
+        List<List<String>> listedLink =  new LinkedList<>();
+        while (tokenStream.incrementToken()) {
+            CharTermAttribute charTerm = tokenStream.getAttribute(CharTermAttribute.class);
+            PositionIncrementAttribute position = tokenStream.getAttribute(PositionIncrementAttribute.class);
+
+
+        }
+
+        return null;
+    }
 }

context/src/main/java/org/apache/lucene/morphology/context/ContextItem.java

@@ -1,11 +1,24 @@
+/**
+ * Copyright 2015 Alexander Kuznetsov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.lucene.morphology.context;
 
-/**
+import java.util.Arrays;
- * Created by alexander on 16.06.15.
+
- */
+public class ContextItem implements Comparable<ContextItem> {
-public class ContextItem {
     String[][] morphInfo;
-    long count = 0;
 
     public ContextItem(String[][] morphInfo) {
         this.morphInfo = morphInfo;
@@ -19,11 +32,49 @@ public class ContextItem {
         this.morphInfo = morphInfo;
     }
 
-    public long getCount() {
+    public int hashCode() {
-        return count;
+        int h = 0;
+        for (String[] m : morphInfo) {
+            for (String s : m) {
+                h = 31 * h + s.hashCode();
+            }
+        }
+        return h;
     }
 
-    public void setCount(long count) {
+
-        this.count = count;
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ContextItem that = (ContextItem) o;
+
+        if (that.morphInfo.length != this.morphInfo.length) {
+            return false;
+        }
+        for (int i = 0; i < morphInfo.length; i++) {
+            if (!Arrays.equals(morphInfo[i], that.morphInfo[i])) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+
+    @Override
+    public int compareTo(ContextItem o) {
+        int i = o.morphInfo.length - morphInfo.length;
+        if (i != 0) return i;
+        for (int j = 0; j < morphInfo.length; j++) {
+            i = o.morphInfo[j].length - morphInfo[j].length;
+            if (i != 0) return i;
+            for (int k = 0; k < morphInfo[j].length; k++) {
+                i = morphInfo[j][k].compareTo(o.morphInfo[j][k]);
+                if (i != 0) return i;
+            }
+        }
+        return 0;
     }
 }

context/src/main/java/org/apache/lucene/morphology/context/ContextStats.java

@@ -1,8 +1,20 @@
+/**
+ * Copyright 2015 Alexander Kuznetsov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.lucene.morphology.context;
 
-/**
- * Created by alexander on 16.06.15.
- */
 public class ContextStats {
     String[] morphInfo;
     double prob;

context/src/main/java/org/apache/lucene/morphology/context/ProbClalucator.java

@@ -1,8 +1,20 @@
+/**
+ * Copyright 2015 Alexander Kuznetsov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.lucene.morphology.context;
 
-/**
- * Created by alexander on 16.06.15.
- */
 public class ProbClalucator {
 
 

context/src/main/java/org/apache/lucene/morphology/context/SimpleTokenizer.java

@@ -1,3 +1,18 @@
+/**
+ * Copyright 2015 Alexander Kuznetsov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.lucene.morphology.context;
 
 import org.apache.lucene.analysis.Tokenizer;
@@ -13,9 +28,6 @@ import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.Set;
 
-/**
- * Created by alexander on 16.06.15.
- */
 public class SimpleTokenizer extends Tokenizer {
 
     private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);

context/src/main/java/org/apache/lucene/morphology/context/StatAnalyzer.java

@@ -0,0 +1,34 @@
+package org.apache.lucene.morphology.context;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.standard.StandardFilter;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * Created by akuznetsov on 6/24/15.
+ */
+public class StatAnalyzer extends Analyzer {
+
+
+
+    @Override
+    protected TokenStreamComponents createComponents(String s) {
+
+        SimpleTokenizer src = new SimpleTokenizer();
+        TokenFilter filter = new StandardFilter(src);
+        filter = new LowerCaseFilter(filter);
+
+        return new TokenStreamComponents(src, filter) {
+            @Override
+            protected void setReader(final Reader reader) throws IOException {
+                super.setReader(reader);
+            }
+        };
+    }
+
+
+}

context/src/test/java/org/apache/lucene/morphology/context/SimpleTokenizerTest.java

@@ -0,0 +1,32 @@
+package org.apache.lucene.morphology.context;
+
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+public class SimpleTokenizerTest {
+
+    @Test
+    public void testSimpleTokenizer() throws IOException {
+        Analyzer statAnalyzer = new StatAnalyzer();
+        InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream("принеси мне вина на новый год? - и что я жду тебя, где вино".getBytes()), "UTF-8");
+
+        TokenStream tokenStream = statAnalyzer.tokenStream(null, reader);
+        tokenStream.reset();
+
+        boolean wordSeen = false;
+        while (tokenStream.incrementToken()) {
+            CharTermAttribute charTerm = tokenStream.getAttribute(CharTermAttribute.class);
+            PositionIncrementAttribute position = tokenStream.getAttribute(PositionIncrementAttribute.class);
+            System.out.println(charTerm.toString());
+        }
+    }
+
+}

etc/header.txt

@@ -1,4 +1,4 @@
-Copyright 2009 Alexander Kuznetsov 
+Copyright ${project.inceptionYear} ${owner}
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

morph/src/main/java/org/apache/lucene/morphology/analyzer/MorphologyAnalyzer.java

@@ -51,25 +51,6 @@ public class MorphologyAnalyzer extends Analyzer {
     protected TokenStreamComponents createComponents(String s) {
 
         StandardTokenizer src = new StandardTokenizer();
-        final PayloadEncoder encoder = new PayloadEncoder() {
-            @Override
-            public BytesRef encode(char[] buffer) {
-                final Float payload = Float.valueOf(new String(buffer));
-                System.out.println(payload);
-                final byte[] bytes = PayloadHelper.encodeFloat(payload);
-                return new BytesRef(bytes, 0, bytes.length);
-            }
-
-            @Override
-            public BytesRef encode(char[] buffer, int offset, int length) {
-
-                final Float payload = Float.valueOf(new String(buffer, offset, length));
-                System.out.println(payload);
-                final byte[] bytes = PayloadHelper.encodeFloat(payload);
-
-                return new BytesRef(bytes, 0, bytes.length);
-            }
-        };
         TokenFilter filter = new StandardFilter(src);
         filter = new LowerCaseFilter(filter);
         filter = new MorphologyFilter(filter, luceneMorph);

pom.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.apache.lucene.morphology</groupId>
     <artifactId>morphology</artifactId>
@@ -64,20 +65,8 @@
             </snapshots>
         </repository>
     </repositories>
-    
+
-    <pluginRepositories>
+
-        <pluginRepository>
-            <id>mc-release</id>
-            <name>maven-license-plugin repository of releases</name>
-            <url>http://mc-repo.googlecode.com/svn/maven2/releases</url>
-            <snapshots>
-                <enabled>false</enabled>
-            </snapshots>
-            <releases>
-                <enabled>true</enabled>
-            </releases>
-        </pluginRepository>
-    </pluginRepositories>
     <build>
         <plugins>
             <plugin>
@@ -98,30 +87,37 @@
                     <target>1.7</target>
                 </configuration>
             </plugin>
-            <plugin>                <!--                 usage: http://code.google.com/p/maven-license-plugin/wiki/HowTo                -->
+            <!--<plugin>                &lt;!&ndash;                 usage: http://code.google.com/p/maven-license-plugin/wiki/HowTo                &ndash;&gt;-->
-                <artifactId>maven-license-plugin</artifactId>
+
-                <groupId>com.mathieucarbou.mojo</groupId>
+                <!--<groupId>com.mycila</groupId>-->
-                <configuration>
+                <!--<artifactId>license-maven-plugin</artifactId>-->
-                    <basedir>${project.parent.basedir}</basedir>
+                <!--<version>2.11</version>-->
-                    <header>etc/header.txt</header>
+
-                    <excludes>
+                <!--<configuration>-->
-                        <exclude>**/*.txt</exclude>
+                    <!--<properties>-->
-                        <exclude>**/*.info</exclude>
+                        <!--<owner>Alexander Kuznetsov</owner>-->
-                        <exclude>**/pom.xml</exclude>
+                        <!--&lt;!&ndash;<email>mathieu.carbou@gmail.com</email>&ndash;&gt;-->
-                    </excludes>
+                    <!--</properties>-->
-                    <includes>
+                    <!--<basedir>${project.parent.basedir}</basedir>-->
-                        <include>**/src/**</include>
+                    <!--<header>etc/header.txt</header>-->
-                    </includes>
+                    <!--<excludes>-->
-                </configuration>
+                        <!--<exclude>**/*.txt</exclude>-->
-                <executions>
+                        <!--<exclude>**/*.info</exclude>-->
-                    <execution>
+                        <!--<exclude>**/pom.xml</exclude>-->
-                        <phase>test</phase>
+                    <!--</excludes>-->
-                        <goals>
+                    <!--<includes>-->
-                            <goal>check</goal>
+                        <!--<include>**/src/**</include>-->
-                        </goals>
+                    <!--</includes>-->
-                    </execution>
+                <!--</configuration>-->
-                </executions>
+                <!--<executions>-->
-            </plugin>
+                    <!--<execution>-->
+                        <!--<phase>test</phase>-->
+                        <!--<goals>-->
+                            <!--<goal>check</goal>-->
+                        <!--</goals>-->
+                    <!--</execution>-->
+                <!--</executions>-->
+            <!--</plugin>-->
         </plugins>
     </build>
     <profiles>
@@ -160,6 +156,6 @@
         <module>dictionary-reader</module>
         <module>russian</module>
         <module>english</module>
-    <module>context</module>
+        <module>context</module>
-  </modules>
+    </modules>
 </project>