Merging code from Dev wordsi into master

.gitignore

@@ -0,0 +1,11 @@
+target/*
+*/target*
+.manager
+*.jar
+*.txt
+*.dat
+*.png
+.*.swp
+*.class
+target
+*~

build.xml

@@ -41,12 +41,16 @@
   <!-- define Maven coordinates, repository url and artifacts name etc -->
   <property name="groupId" value="edu.ucla.sspace" />
   <property name="artifactId" value="sspace" />
-  <property name="maven-repository-url" value="https://oss.sonatype.org/service/local/staging/deploy/maven2/" />
-  <property name="maven-repository-id" value="sonatype-nexus-staging" />
   <property name="maven-jar" value="${dist.dir}/${artifactId}-${version}.jar" />
   <property name="maven-javadoc-jar" value="${dist.dir}/${artifactId}-${version}-javadoc.jar" />
   <property name="maven-sources-jar" value="${dist.dir}/${artifactId}-${version}-src.jar" />
 
+  <property name="maven-snapshots-repository-id" value="sonatype-nexus-snapshots" />
+  <property name="maven-snapshots-repository-url" value="https://oss.sonatype.org/content/repositories/snapshots/" />
+  <property name="maven-staging-repository-id" value="sonatype-nexus-staging" />
+  <property name="maven-staging-repository-url" value="https://oss.sonatype.org/service/local/staging/deploy/maven2/" />
+
+
   <path id="classpath">
     <fileset dir="${lib.dir}" includes="**/*.jar"/>
   </path>
@@ -93,7 +97,7 @@
     <!-- Compile all of the classes and put the results in the classes
 	 directory -->
      <javac srcdir="./" destdir="classes" classpathref="classpath" 
-	    debug="true" debuglevel="lines,source">
+	    debug="true" nowarn="yes" debuglevel="lines,source">
        <compilerarg value="-Xlint"/>
        <compilerarg value="-g"/>
        <include name="src/**"/>
@@ -168,24 +172,37 @@
   </target>
 
   <!--
-       Deploy the distribution jars to the maven repository.
+       Deploy a snapshot of the distribution jars to the maven repository.
+  -->
+  <target name="deploy" depends="dist" description="deploy snapshot version to Maven snapshot repository">
+    <artifact:mvn>
+      <arg value="org.apache.maven.plugins:maven-deploy-plugin:2.6:deploy-file" />
+      <arg value="-Durl=${maven-snapshots-repository-url}" />
+      <arg value="-DrepositoryId=${maven-snapshots-repository-id}" />
+      <arg value="-DpomFile=pom.xml" />
+      <arg value="-Dfile=${maven-jar}" />
+    </artifact:mvn>
+  </target>
+
+  <!--
+       Stage the distribution jars to the maven repository.
   -->
-  <target name="deploy" depends="dist" description="deploy to Nexus">
+  <target name="stage" depends="dist" description="deploy to Nexus">
     <!-- sign and deploy the main artifact -->
     <artifact:mvn>
-      <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.1:sign-and-deploy-file" />
-      <arg value="-Durl=${maven-repository-url}" />
-      <arg value="-DrepositoryId=${maven-repository-id}" />
+      <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.3:sign-and-deploy-file" />
+      <arg value="-Durl=${maven-staging-repository-url}" />
+      <arg value="-DrepositoryId=${maven-staging-repository-id}" />
       <arg value="-DpomFile=pom.xml" />
       <arg value="-Dfile=${maven-jar}" />
       <arg value="-Pgpg" />
     </artifact:mvn>
 
     <!-- sign and deploy the sources artifact -->
     <artifact:mvn>
-      <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.1:sign-and-deploy-file" />
-      <arg value="-Durl=${maven-repository-url}" />
-      <arg value="-DrepositoryId=${maven-repository-id}" />
+      <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.3:sign-and-deploy-file" />
+      <arg value="-Durl=${maven-stagingrepository-url}" />
+      <arg value="-DrepositoryId=${maven-staging-repository-id}" />
       <arg value="-DpomFile=pom.xml" />
       <arg value="-Dfile=${maven-sources-jar}" />
       <arg value="-Dclassifier=sources" />
@@ -194,9 +211,9 @@
 
     <!-- sign and deploy the javadoc artifact -->
     <artifact:mvn>
-      <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.1:sign-and-deploy-file" />
-      <arg value="-Durl=${maven-repository-url}" />
-      <arg value="-DrepositoryId=${maven-repository-id}" />
+      <arg value="org.apache.maven.plugins:maven-gpg-plugin:1.3:sign-and-deploy-file" />
+      <arg value="-Durl=${maven-staging-repository-url}" />
+      <arg value="-DrepositoryId=${maven-staging-repository-id}" />
       <arg value="-DpomFile=pom.xml" />
       <arg value="-Dfile=${maven-javadoc-jar}" />
       <arg value="-Dclassifier=javadoc" />

data/english-stop-words-large.txt

@@ -590,3 +590,70 @@ u
 's
 /
 dr.
+''
+``
+(
+)
+--
+&
+%
+n't
+mr.
+{
+}
+ms.
+mrs.
+'ll
+'re
+[
+]
+...
+-
+'ve
+/
+\
+&quot
+st
+ii
+&gt
+&lt
+.e
+j.
+de
+a.
+b.
+c.
+d.
+e.
+f.
+g.
+h.
+i.
+j.
+k.
+l.
+m.
+n.
+o.
+p.
+q.
+r.
+s.
+t.
+u.
+v.
+w.
+x.
+y.
+z.
+'d
+ve
+and/or
+ll
+'ll
+-lrb-
+-rrb-
+`
+\/
+''
+% %

pom.xml

@@ -5,7 +5,7 @@
   <modelVersion>4.0.0</modelVersion>
 
   <groupId>edu.ucla.sspace</groupId>
-  <artifactId>sspace</artifactId>
+  <artifactId>sspace-wordsi</artifactId>
   <packaging>jar</packaging>
   <name>S-Space Package</name>
   <version>2.0.3</version>
@@ -79,6 +79,11 @@
   </distributionManagement>
 
   <dependencies>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-math</artifactId>
+      <version>2.2</version>
+    </dependency>
     <dependency>
       <groupId>net.sf.trove4j</groupId>
       <artifactId>trove4j</artifactId>
@@ -90,6 +95,16 @@
       <version>4.8</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.googlecode.netlib-java</groupId>
+      <artifactId>netlib-java</artifactId>
+      <version>0.9.3</version>
+    </dependency>
+    <dependency>
+      <groupId>net.sourceforge.f2j</groupId>
+      <artifactId>arpack_combined_all</artifactId>
+      <version>0.1</version>
+    </dependency>
   </dependencies>
 
   <build>

src/main/java/edu/ucla/sspace/basis/FilteredStringBasisMapping.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2011, Lawrence Livermore National Security, LLC. Produced at
+ * the Lawrence Livermore National Laboratory. Written by Keith Stevens,
+ * [email protected] OCEC-10-073 All rights reserved. 
+ *
+ * This file is part of the S-Space package and is covered under the terms and
+ * conditions therein.
+ *
+ * The S-Space package is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation and distributed hereunder to you.
+ *
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND NO REPRESENTATIONS OR WARRANTIES,
+ * EXPRESS OR IMPLIED ARE MADE.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, WE MAKE
+ * NO REPRESENTATIONS OR WARRANTIES OF MERCHANT- ABILITY OR FITNESS FOR ANY
+ * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE OR DOCUMENTATION
+ * WILL NOT INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER
+ * RIGHTS.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package edu.ucla.sspace.basis;
+
+import java.util.Set;
+
+
+/**
+ * This {@link FilteredStringBasisMapping} allows a user to specify a set of
+ * tokens that should be excluded automatically from the basis mapping.  Any
+ * calles to {@code getDimension} for words in this set will automatically
+ * return {@code -1}.
+ *
+ * @author Keith Stevens
+ */
+public class FilteredStringBasisMapping
+        extends AbstractBasisMapping<String, String> {
+
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * The set of excluded words.
+     */
+    private final Set<String> excludedWords;
+
+    /**
+     * Creates a new {@link FilteredStringBasisMapping} where the words in
+     * {@code excludedWords} will never receive a dimension in this mapping.
+     */
+    public FilteredStringBasisMapping(Set<String> excludedWords) {
+        this.excludedWords = excludedWords;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public int getDimension(String key) {
+        String[] parts = key.split("-");
+        String base = (parts.length == 0) ? key : parts[0];
+        return excludedWords.contains(base) ? -1 : getDimensionInternal(key);
+    }
+}