Merge remote-tracking branch 'refs/remotes/origin/master'

Author: manning

itest/src/edu/stanford/nlp/pipeline/POSTaggerBenchmarkITest.java

@@ -82,7 +82,7 @@ public void testEnglishBiDirectionalPOSModelAccuracy() {
     String englishPOSTestPath = "/u/nlp/data/pos-tagger/english/test-wsj-22-24";
     List<String> sentences = readInPOSData(englishPOSTestPath);
     double ENGLISH_BIDIRECTIONAL_TOKEN_ACCURACY = .972;
-    double ENGLISH_BIDIRECTIONAL_SENTENCE_ACCURACY = .564;
+    double ENGLISH_BIDIRECTIONAL_SENTENCE_ACCURACY = .563;
     runPOSTest(sentences, "_", englishBiDirectionalPipeline,
         ENGLISH_BIDIRECTIONAL_TOKEN_ACCURACY, ENGLISH_BIDIRECTIONAL_SENTENCE_ACCURACY,
         "English BiDirectional", false);

lib/lucene-analyzers-common-4.10.3.jar

@@ -57,25 +57,25 @@
     <dependency>
       <groupId>org.apache.lucene</groupId>
       <artifactId>lucene-queryparser</artifactId>
-      <version>4.10.3</version>
+      <version>7.5.0</version>
     </dependency>
 
     <dependency>
       <groupId>org.apache.lucene</groupId>
       <artifactId>lucene-analyzers-common</artifactId>
-      <version>4.10.3</version>
+      <version>7.5.0</version>
     </dependency>
 
     <dependency>
       <groupId>org.apache.lucene</groupId>
       <artifactId>lucene-queries</artifactId>
-      <version>4.10.3</version>
+      <version>7.5.0</version>
     </dependency>
 
     <dependency>
       <groupId>org.apache.lucene</groupId>
       <artifactId>lucene-core</artifactId>
-      <version>4.10.3</version>
+      <version>7.5.0</version>
     </dependency>
 
     <dependency>

lib/lucene-analyzers-common-7.5.0.jar

@@ -68,7 +68,7 @@ public class LuceneSentenceIndex<E extends Pattern> extends SentenceIndex<E> {
 //  }
 
   //StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_42);
-  IndexWriterConfig iwc=new IndexWriterConfig(Version.LUCENE_42, analyzer);
+  IndexWriterConfig iwc=new IndexWriterConfig(analyzer);
   DirectoryReader reader = null;
   IndexSearcher searcher;
   ProtobufAnnotationSerializer p = new ProtobufAnnotationSerializer();
@@ -82,7 +82,7 @@ public LuceneSentenceIndex(Properties props, Set<String> stopWords, String index
 
 
   void setIndexReaderSearcher() throws IOException {
-    FSDirectory index = FSDirectory.open(indexDir);
+    FSDirectory index = FSDirectory.open(indexDir.toPath());
     if(reader == null){
       reader = DirectoryReader.open(index);
       searcher = new IndexSearcher(reader);
@@ -126,16 +126,17 @@ void setIndexReaderSearcher() throws IOException {
    * **/
   Set<String> queryIndexGetSentences(CollectionValuedMap<String, String> words) throws IOException, ParseException {
     setIndexReaderSearcher();
-    BooleanQuery query = new BooleanQuery();
+    BooleanQuery.Builder builder = new BooleanQuery.Builder();
     String pkey  = Token.getKeyForClass(PatternsAnnotations.ProcessedTextAnnotation.class);
 
     for(Map.Entry<String, Collection<String>> en: words.entrySet()){
       boolean processedKey = en.getKey().equals(pkey);
       for(String en2: en.getValue()){
           if(!processedKey || !stopWords.contains(en2.toLowerCase()))
-            query.add(new BooleanClause(new TermQuery(new Term(en.getKey(), en2)), BooleanClause.Occur.MUST));
+            builder.add(new BooleanClause(new TermQuery(new Term(en.getKey(), en2)), BooleanClause.Occur.MUST));
       }
     }
+    BooleanQuery query = builder.build();
     //query.add(new BooleanClause(new TermQuery(new Term("textannotation","sonal")), BooleanClause.Occur.MUST));
 
 //    String queryStr = "";
@@ -304,7 +305,7 @@ public void update(List<CoreLabel> tokens, String sentid) {
 
   void setIndexWriter()  {try{
     if(indexWriter == null){
-      dir = FSDirectory.open(indexDir);
+      dir = FSDirectory.open(indexDir.toPath());
       Redwood.log(Redwood.DBG, "Updating lucene index at " + indexDir);
       indexWriter = new IndexWriter(dir, iwc);
     }}catch(IOException e){

lib/lucene-core-4.10.3.jar

@@ -33,7 +33,7 @@ public class PatternsForEachTokenLucene<E extends Pattern> extends PatternsForEa
 
   static Analyzer analyzer = new KeywordAnalyzer();
 
-  static IndexWriterConfig iwc=new IndexWriterConfig(Version.LUCENE_42, analyzer);
+  static IndexWriterConfig iwc=new IndexWriterConfig(analyzer);
   static DirectoryReader reader = null;
   static IndexSearcher searcher;
 
@@ -77,7 +77,7 @@ public PatternsForEachTokenLucene(Properties props, Map<String, Map<Integer, Set
 
   public void checkClean(){
     try {
-      dir = FSDirectory.open(indexDir);
+      dir = FSDirectory.open(indexDir.toPath());
     CheckIndex checkIndex = new CheckIndex(dir);
     CheckIndex.Status status = checkIndex.checkIndex();
     assert (status.clean) : "index is not clean";
@@ -98,7 +98,7 @@ public void setupSearch(){
 
   static synchronized void setIndexReaderSearcher() {
     try{
-    FSDirectory index = NIOFSDirectory.open(indexDir);
+    FSDirectory index = NIOFSDirectory.open(indexDir.toPath());
     if(reader == null){
       reader = DirectoryReader.open(index);
       searcher = new IndexSearcher(reader);
@@ -166,7 +166,7 @@ public void addPatterns(Map<String, Map<Integer, Set<E>>> pats) {
   static synchronized void setIndexWriter()  {
     try{
     if(!openIndexWriter.get()){
-      dir = FSDirectory.open(indexDir);
+      dir = FSDirectory.open(indexDir.toPath());
       Redwood.log(Redwood.DBG, "Updating lucene index at " + indexDir);
       indexWriter = new IndexWriter(dir, iwc);
       openIndexWriter.set(true);

lib/lucene-core-7.5.0.jar

@@ -28,7 +28,7 @@
 <nav class="navbar navbar-default navbar-static-top">
   <div class="container">
     <div class="navbar-header">
-      <a class="navbar-brand" href="https://nlp.stanford.edu/software/corenlp.shtml">Stanford CoreNLP 3.9.1 (updated 2018/04/05)</a>
+      <a class="navbar-brand" href="https://stanfordnlp.github.io/CoreNLP/">Stanford CoreNLP 3.9.1 (updated 2018/04/05)</a>
     </div>
   </div>
 </nav>

lib/lucene-demo-4.10.3.jar

@@ -1,6 +1,7 @@
 package edu.stanford.nlp.util;
 
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.IndexOptions;
 
 /** An easy way to access types of fields instead of setting variables up every time.
  * Copied from KBPFieldType written by Angel.
@@ -24,34 +25,34 @@ public class LuceneFieldType {
   public static final FieldType NOT_INDEXED = new FieldType();
 
   static {
-    ANALYZED_NOT_STORED.setIndexed(true);
+    ANALYZED_NOT_STORED.setIndexOptions(IndexOptions.DOCS);
     ANALYZED_NOT_STORED.setTokenized(true);
     ANALYZED_NOT_STORED.setStored(false);
     ANALYZED_NOT_STORED.setStoreTermVectors(true);
     ANALYZED_NOT_STORED.setStoreTermVectorPositions(true);
     ANALYZED_NOT_STORED.freeze();
 
-    ANALYZED.setIndexed(true);
+    ANALYZED.setIndexOptions(IndexOptions.DOCS);
     ANALYZED.setTokenized(true);
     ANALYZED.setStored(true);
     ANALYZED.setStoreTermVectors(true);
     ANALYZED.setStoreTermVectorPositions(true);
     ANALYZED.freeze();
 
-    ANALYZED_NO_POSITION.setIndexed(true);
+    ANALYZED_NO_POSITION.setIndexOptions(IndexOptions.DOCS);
     ANALYZED_NO_POSITION.setTokenized(true);
     ANALYZED_NO_POSITION.setStoreTermVectors(true);
     ANALYZED_NO_POSITION.setStoreTermVectorPositions(false);
     ANALYZED_NO_POSITION.freeze();
 
-    NOT_ANALYZED.setIndexed(true);
+    NOT_ANALYZED.setIndexOptions(IndexOptions.DOCS);
     NOT_ANALYZED.setTokenized(false);
     NOT_ANALYZED.setStored(true);
     NOT_ANALYZED.setStoreTermVectors(false);
     NOT_ANALYZED.setStoreTermVectorPositions(false);
     NOT_ANALYZED.freeze();
 
-    NOT_INDEXED.setIndexed(false);
+    NOT_INDEXED.setIndexOptions(IndexOptions.NONE);
     NOT_INDEXED.setTokenized(false);
     NOT_INDEXED.setStored(true);
     NOT_INDEXED.setStoreTermVectors(false);