get tests passing again; fix some jirasearch issues

Author: mikemccand

build.py

@@ -24,8 +24,8 @@
 
 testDeps = [
   ('de.thetaphi', 'forbiddenapis', '2.2'),
-  ('com.carrotsearch.randomizedtesting', 'junit4-ant', '2.0.13'),
-  ('com.carrotsearch.randomizedtesting', 'randomizedtesting-runner', '2.3.4'),
+  ('com.carrotsearch.randomizedtesting', 'junit4-ant', '2.4.0'),
+  ('com.carrotsearch.randomizedtesting', 'randomizedtesting-runner', '2.4.0'),
   ('junit', 'junit', '4.10')
   ]
 
@@ -185,7 +185,7 @@ def run(self):
                 s = s[:s.find('(')]
                 message('\nNOTE: test case done: %.3f sec for %s\n' % (event[1]['executionTime']/1000., s))
             elif event[0] in ('TEST_FAILURE', 'SUITE_FAILURE'):
-              details = event[1]['failure']
+              details = event[1]
               s = '\n!! %s.%s FAILED !!:\n\n' % (job[25:], testCaseName)
               s += ''.join(pendingOutput)
               if 'message' in details:
@@ -199,7 +199,8 @@ def run(self):
             elif event[0] == 'TEST_STARTED':
               self.testCount += 1
               testCaseFailed = False
-              testCaseName = event[1]['description']
+              #testCaseName = event[1]['description']
+              testCaseName = event[1]
               i = testCaseName.find('#')
               j = testCaseName.find('(')
               testCaseName = testCaseName[i+1:j]

src/java/org/apache/lucene/server/FromFileTermFreqIterator.java

@@ -35,6 +35,7 @@ public class FromFileTermFreqIterator implements InputIterator, Closeable {
   private final BufferedReader reader;
   private long weight;
   private int lineCount;
+  final boolean hasContexts;
 
   /** How many suggestions were found. */
   public int suggestCount;
@@ -43,13 +44,14 @@ public class FromFileTermFreqIterator implements InputIterator, Closeable {
   private final Set<BytesRef> contexts = new HashSet<>();
 
   /** Sole constructor. */
-  public FromFileTermFreqIterator(File sourceFile) throws IOException {
+  public FromFileTermFreqIterator(File sourceFile, boolean hasContexts) throws IOException {
     reader = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile), "UTF-8"), 65536);
+    this.hasContexts = hasContexts;
   }
 
   @Override
   public boolean hasContexts() {
-    return true;
+    return hasContexts;
   }
 
   @Override
@@ -93,25 +95,25 @@ public BytesRef next() {
 
       int spot3 = line.indexOf('\u001f', spot2+1);
       if (spot3 == -1) {
-        throw new RuntimeException("line " + lineCount + " is malformed");
-      }
-      
-      extra = new BytesRef(line.substring(spot2+1, spot3));
+        extra = new BytesRef(line.substring(spot2+1));
+      } else {
+        extra = new BytesRef(line.substring(spot2+1, spot3));
 
-      contexts.clear();
+        contexts.clear();
 
-      int upto = spot3+1;
-      while (true ){
-        int nextUpto = line.indexOf('\u001f', upto);
-        if (nextUpto == -1) {
-          contexts.add(new BytesRef(line.substring(upto)));
-          break;
-        } else {
-          contexts.add(new BytesRef(line.substring(upto, nextUpto)));
-          upto = nextUpto+1;
+        int upto = spot3+1;
+        while (true) {
+          int nextUpto = line.indexOf('\u001f', upto);
+          if (nextUpto == -1) {
+            contexts.add(new BytesRef(line.substring(upto)));
+            break;
+          } else {
+            contexts.add(new BytesRef(line.substring(upto, nextUpto)));
+            upto = nextUpto+1;
+          }
         }
+        //System.out.println("CONTEXTS: " + text.utf8ToString() + " -> " + contexts);
       }
-      //System.out.println("CONTEXTS: " + text.utf8ToString() + " -> " + contexts);
 
       return text;
     }

src/java/org/apache/lucene/server/IndexState.java

@@ -92,7 +92,7 @@
 import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester;
 import org.apache.lucene.server.handlers.BuildSuggestHandler;
 import org.apache.lucene.server.handlers.LiveSettingsHandler;
-import org.apache.lucene.server.handlers.RegisterFieldHandler;
+import org.apache.lucene.server.handlers.RegisterFieldsHandler;
 import org.apache.lucene.server.handlers.SettingsHandler;
 import org.apache.lucene.server.params.BooleanType;
 import org.apache.lucene.server.params.FloatType;
@@ -798,8 +798,8 @@ public synchronized void load(JSONObject o) throws IOException {
       JSONObject fieldsState = (JSONObject) o.get("fields");
       JSONObject top = new JSONObject();
       top.put("fields", fieldsState);
-      Request r = new Request(null, null, top, RegisterFieldHandler.TYPE);
-      FinishRequest fr = ((RegisterFieldHandler) globalState.getHandler("registerFields")).handle(this, r, null);
+      Request r = new Request(null, null, top, RegisterFieldsHandler.TYPE);
+      FinishRequest fr = ((RegisterFieldsHandler) globalState.getHandler("registerFields")).handle(this, r, null);
       assert !Request.anythingLeft(top): top;
       fr.finish();
 

src/java/org/apache/lucene/server/Server.java

@@ -82,7 +82,7 @@
 import org.apache.lucene.server.handlers.NewNRTPointHandler;
 import org.apache.lucene.server.handlers.NodeToNodeHandler;
 import org.apache.lucene.server.handlers.RefreshHandler;
-import org.apache.lucene.server.handlers.RegisterFieldHandler;
+import org.apache.lucene.server.handlers.RegisterFieldsHandler;
 import org.apache.lucene.server.handlers.ReleaseSnapshotHandler;
 import org.apache.lucene.server.handlers.RollbackHandler;
 import org.apache.lucene.server.handlers.Search2Handler;
@@ -662,7 +662,7 @@ public Server(String nodeName, Path globalStateDir, int backlog, int threadCount
     globalState.addHandler("indexStatus", new IndexStatusHandler(globalState));
     globalState.addHandler("linkNode", new LinkNodeHandler(globalState));
     globalState.addHandler("liveSettings", new LiveSettingsHandler(globalState));
-    globalState.addHandler("registerFields", new RegisterFieldHandler(globalState));
+    globalState.addHandler("registerFields", new RegisterFieldsHandler(globalState));
     globalState.addHandler("releaseSnapshot", new ReleaseSnapshotHandler(globalState));
     globalState.addHandler("search", new SearchHandler(globalState));
     globalState.addHandler("search2", new Search2Handler(globalState));

src/java/org/apache/lucene/server/handlers/AnalysisHandler.java

@@ -42,7 +42,7 @@ public class AnalysisHandler extends Handler {
   private final static StructType TYPE = new StructType(
                                                         new Param("indexName", "Index Name", new StringType()),
                                                         new Param("text", "Text to analyze", new StringType()),
-                                                        new Param("analyzer", "Analyzer", RegisterFieldHandler.ANALYZER_TYPE));
+                                                        new Param("analyzer", "Analyzer", RegisterFieldsHandler.ANALYZER_TYPE));
 
   /** Sole constructor. */
   public AnalysisHandler(GlobalState state) {
@@ -64,7 +64,7 @@ public FinishRequest handle(final IndexState state, final Request r, Map<String,
     final String text = r.getString("text");
 
     // TODO: allow passing field name in, and we use its analyzer?
-    final Analyzer a = RegisterFieldHandler.getAnalyzer(state, r, "analyzer");
+    final Analyzer a = RegisterFieldsHandler.getAnalyzer(state, r, "analyzer");
 
     return new FinishRequest() {
       @Override

src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java

@@ -63,7 +63,7 @@
 import net.minidev.json.JSONValue;
 import net.minidev.json.parser.ParseException;
 
-import static org.apache.lucene.server.handlers.RegisterFieldHandler.ANALYZER_TYPE;
+import static org.apache.lucene.server.handlers.RegisterFieldsHandler.ANALYZER_TYPE;
 
 /** Handles {@code buildSuggest}. */
 public class BuildSuggestHandler extends Handler {
@@ -103,6 +103,7 @@ public class BuildSuggestHandler extends Handler {
         new Param("source", "Where to get suggestions from",
             new StructType(
                   new Param("localFile", "Local file (to the server) to read suggestions + weights from; format is weight U+001F suggestion U+001F payload, one per line, with suggestion UTF-8 encoded.  If this option is used then searcher, suggestField, weightField/Expression, payloadField should not be specified.", new StringType()),
+                  new Param("hasContexts", "True if this file provides per-suggestion contexts.", new BooleanType(), true),
                   new Param("searcher", "Specific searcher version to use for pull suggestions to build.  There are three different ways to specify a searcher version.",
                             SearchHandler.SEARCHER_VERSION_TYPE),
                   new Param("suggestField", "Field (from stored documents) containing the suggestion text", new StringType()),
@@ -142,6 +143,7 @@ public void load(IndexState indexState, JSONObject saveState) throws IOException
       Request source = r.getStruct("source");
       if (source.hasParam("localFile")) {
         source.getString("localFile");
+        source.getBoolean("hasContexts");
       } else {
         Request searcher = source.getStruct("searcher");
         if (searcher.hasParam("indexGen")) {
@@ -198,10 +200,10 @@ private Lookup getSuggester(IndexState indexState, String suggestName, Request r
     // nocommit allow passing a field name, and we use that
     // field name's analyzer?
     if (r.hasParam("analyzer")) {
-      indexAnalyzer = queryAnalyzer = RegisterFieldHandler.getAnalyzer(indexState, r, "analyzer");
+      indexAnalyzer = queryAnalyzer = RegisterFieldsHandler.getAnalyzer(indexState, r, "analyzer");
     } else {
-      indexAnalyzer = RegisterFieldHandler.getAnalyzer(indexState, r, "indexAnalyzer");
-      queryAnalyzer = RegisterFieldHandler.getAnalyzer(indexState, r, "queryAnalyzer");
+      indexAnalyzer = RegisterFieldsHandler.getAnalyzer(indexState, r, "indexAnalyzer");
+      queryAnalyzer = RegisterFieldsHandler.getAnalyzer(indexState, r, "queryAnalyzer");
     }
     if (indexAnalyzer == null) {
       r.fail("analyzer", "analyzer or indexAnalyzer must be specified");
@@ -420,10 +422,11 @@ public FinishRequest handle(final IndexState indexState, final Request r, Map<St
       if (!localFile.canRead()) {
         r.fail("localFile", "cannot read file");
       }
+      boolean hasContexts = source.getBoolean("hasContexts");
       searcher = null;
       // Pull suggestions from local file:
       try {
-        iterator = new FromFileTermFreqIterator(localFile);
+        iterator = new FromFileTermFreqIterator(localFile, hasContexts);
       } catch (IOException ioe) {
         r.fail("localFile", "cannot open file", ioe);
       }

src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java src/java/org/apache/lucene/server/handlers/RegisterFieldsHandler.java

@@ -106,10 +106,8 @@
 import net.minidev.json.JSONObject;
 import net.minidev.json.JSONValue;
 
-// nocommit rename to RegisterFieldsHandler:
-
 /** Handles {@code registerFields}. */
-public class RegisterFieldHandler extends Handler {
+public class RegisterFieldsHandler extends Handler {
 
   private final static List<Object> DEFAULT_ENGLISH_STOP_WORDS = new ArrayList<Object>();
   static {
@@ -347,7 +345,7 @@ public class RegisterFieldHandler extends Handler {
             new StructType(new Param("*", "Register this field name with the provided type.  Note that the field name must be of the form [a-zA-Z_][a-zA-Z_0-9]*.  You can register multiple fields in one request.", FIELD_TYPE))));
 
   /** Sole constructor. */
-  public RegisterFieldHandler(GlobalState state) {
+  public RegisterFieldsHandler(GlobalState state) {
     super(state);
   }
 
@@ -450,11 +448,9 @@ private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,Field
     boolean sorted = f.getBoolean("sort");
     boolean grouped = f.getBoolean("group");
 
-    boolean stored;
+    Boolean stored = null;
     if (f.hasParam("store")) {
       stored = f.getBoolean("store");
-    } else {
-      stored = false;
     }
 
     // TODO: current we only highlight using
@@ -481,6 +477,13 @@ private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,Field
       }
     }
 
+    // if stored was unspecified and we are highlighting, turn it on:
+    if (highlighted) {
+      if (stored == null) {
+        stored = true;
+      }
+    }
+    
     String dateTimeFormat = null;
 
     // System.out.println("NAME: " + name + " type: " + type + " hightlight: " + highlighted);
@@ -498,7 +501,9 @@ private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,Field
         ft.setDocValuesType(DocValuesType.BINARY);
       }
       if (highlighted) {
-        stored = true;
+        if (stored == false) {
+          f.fail("store", "store=false is not allowed when highlight=true");
+        }
         ft.setStored(true);
         ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
       } else {
@@ -510,7 +515,16 @@ private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,Field
       if (f.hasParam("analyzer")) {
         f.fail("analyzer", "no analyzer allowed with atom (it's hardwired to KeywordAnalyzer internally)");
       }
-      ft.setIndexOptions(IndexOptions.DOCS);
+      if (highlighted) {
+        if (stored == false) {
+          f.fail("store", "store=false is not allowed when highlight=true");
+        }
+        // nocommit need test highlighting atom fields
+        ft.setStored(true);
+        ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+      } else {
+        ft.setIndexOptions(IndexOptions.DOCS);
+      }
       ft.setOmitNorms(true);
       ft.setTokenized(false);
       if (sorted || grouped) {
@@ -575,7 +589,7 @@ private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,Field
       break;
 
     case LAT_LON:
-      if (stored) {
+      if (stored == Boolean.TRUE) {
         f.fail("stored", "latlon fields cannot be stored");
       }
       ft.setDimensions(2, Integer.BYTES);
@@ -613,14 +627,10 @@ private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,Field
       throw new AssertionError("unhandled type \"" + type + "\"");
     }
 
-    // System.out.println("REGISTER: " + name + " ft=" + ft);
-
     // nocommit InetAddressPoint, BiggishInteger
 
-    if (stored) {
+    if (stored == Boolean.TRUE) {
       ft.setStored(true);
-    } else if (highlighted) {
-      f.fail("store", "store=false is not allowed when highlight=true");
     }
 
     boolean usePoints = false;

src/java/org/apache/lucene/server/handlers/UpdateSuggestHandler.java

@@ -29,11 +29,13 @@
 import org.apache.lucene.server.FromFileTermFreqIterator;
 import org.apache.lucene.server.GlobalState;
 import org.apache.lucene.server.IndexState;
+import org.apache.lucene.server.params.BooleanType;
 import org.apache.lucene.server.params.Param;
 import org.apache.lucene.server.params.Request;
 import org.apache.lucene.server.params.StringType;
 import org.apache.lucene.server.params.StructType;
 import org.apache.lucene.util.BytesRef;
+
 import net.minidev.json.JSONObject;
 
 /** Handles {@code updateSuggest}. */
@@ -47,7 +49,8 @@ public class UpdateSuggestHandler extends Handler {
         // nocommit also allow pulling from index/expressions, like BuildSuggest:
         new Param("source", "Where to get suggestions from",
             new StructType(
-                new Param("localFile", "Local file (to the server) to read suggestions + weights from; format is weight U+001F suggestion U+001F payload, one per line, with suggestion UTF-8 encoded.  If this option is used then searcher, suggestField, weightField/Expression, payloadField should not be specified.", new StringType()))));
+                           new Param("localFile", "Local file (to the server) to read suggestions + weights from; format is weight U+001F suggestion U+001F payload, one per line, with suggestion UTF-8 encoded.  If this option is used then searcher, suggestField, weightField/Expression, payloadField should not be specified.", new StringType()),
+                           new Param("hasContexts", "True if this suggester file provides contexts", new BooleanType(), true))));
 
   /** Sole constructor. */
   public UpdateSuggestHandler(GlobalState state) {
@@ -80,13 +83,14 @@ public FinishRequest handle(final IndexState state, final Request r, Map<String,
 
     Request r2 = r.getStruct("source");
     final File localFile = new File(r2.getString("localFile"));
+    final boolean hasContexts = r2.getBoolean("hasContexts");
 
     return new FinishRequest() {
 
       @Override
       public String finish() throws IOException {
 
-        InputIterator iterator = new FromFileTermFreqIterator(localFile);
+        InputIterator iterator = new FromFileTermFreqIterator(localFile, hasContexts);
         boolean hasPayloads = iterator.hasPayloads();
         int count = 0;
         while (true) {

src/test/org/apache/lucene/server/TestSuggest.java

@@ -61,7 +61,7 @@ public void testAnalyzingSuggest() throws Exception {
     out.write("5\u001fthe time is now\u001ffoobar\n");
     out.close();
 
-    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.toAbsolutePath() + "'}, class: 'AnalyzingSuggester', suggestName: 'suggest', indexAnalyzer: EnglishAnalyzer, queryAnalyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, PorterStem]]}}");
+    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.toAbsolutePath() + "', hasContexts: false}, class: 'AnalyzingSuggester', suggestName: 'suggest', indexAnalyzer: EnglishAnalyzer, queryAnalyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, PorterStem]]}}");
     assertEquals(5, result.get("count"));
     //commit();
 
@@ -203,7 +203,7 @@ public void testFuzzySuggest() throws Exception {
     out.write("15\u001flove lost\u001ffoobar\n");
     out.close();
 
-    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.toAbsolutePath() + "'}, class: 'FuzzySuggester', suggestName: 'suggest3', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
+    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.toAbsolutePath() + "', hasContexts: false}, class: 'FuzzySuggester', suggestName: 'suggest3', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
     assertEquals(1, result.get("count"));
     //commit();