Merge branch 'master' of jamie.stanford.edu:/u/nlp/git/javanlp

Author: Angel Chang

README.md

@@ -38,13 +38,13 @@ At present [the current released version of the code](https://stanfordnlp.github
 #### Build with Maven
 
 1. Make sure you have Maven installed, details here: [https://maven.apache.org/](https://maven.apache.org/)
-2. If you run this command in the CoreNLP directory: `mvn package` , it should run the tests and build this jar file: `CoreNLP/target/stanford-corenlp-3.7.0.jar`
+2. If you run this command in the CoreNLP directory: `mvn package` , it should run the tests and build this jar file: `CoreNLP/target/stanford-corenlp-3.9.2.jar`
 3. When using the latest version of the code make sure to download the latest versions of the [corenlp-models](http://nlp.stanford.edu/software/stanford-corenlp-models-current.jar), [english-models](http://nlp.stanford.edu/software/stanford-english-corenlp-models-current.jar), and [english-models-kbp](http://nlp.stanford.edu/software/stanford-english-kbp-corenlp-models-current.jar) and include them in your CLASSPATH.  If you are processing languages other than English, make sure to download the latest version of the models jar for the language you are interested in.  
-4. If you want to use Stanford CoreNLP as part of a Maven project you need to install the models jars into your Maven repository.  Below is a sample command for installing the Spanish models jar.  For other languages just change the language name in the command.  To install `stanford-corenlp-models-current.jar` you will need to set `-Dclassifier=models`.  Here is the sample command for Spanish: `mvn install:install-file -Dfile=/location/of/stanford-spanish-corenlp-models-current.jar -DgroupId=edu.stanford.nlp -DartifactId=stanford-corenlp -Dversion=3.9.1 -Dclassifier=models-spanish -Dpackaging=jar` 
+4. If you want to use Stanford CoreNLP as part of a Maven project you need to install the models jars into your Maven repository.  Below is a sample command for installing the Spanish models jar.  For other languages just change the language name in the command.  To install `stanford-corenlp-models-current.jar` you will need to set `-Dclassifier=models`.  Here is the sample command for Spanish: `mvn install:install-file -Dfile=/location/of/stanford-spanish-corenlp-models-current.jar -DgroupId=edu.stanford.nlp -DartifactId=stanford-corenlp -Dversion=3.9.2 -Dclassifier=models-spanish -Dpackaging=jar` 
 
 ### Useful resources
 
-You can find releases of Stanford CoreNLP on [Maven Central](https://search.maven.org/#artifactdetails%7Cedu.stanford.nlp%7Cstanford-corenlp%7C3.7.0%7Cjar).
+You can find releases of Stanford CoreNLP on [Maven Central](https://search.maven.org/artifact/edu.stanford.nlp/stanford-corenlp/3.9.2/jar).
 
 You can find more explanation and documentation on [the Stanford CoreNLP homepage](http://stanfordnlp.github.io/CoreNLP/).
 

src/edu/stanford/nlp/process/PTBLexer.flex

@@ -249,7 +249,7 @@ import edu.stanford.nlp.util.logging.Redwood;
 
 
   /** Turn on to find out how things were tokenized. */
-  private static final boolean DEBUG = false;
+  private static final boolean DEBUG = true;
 
   /** A logger for this class */
   private static final Redwood.RedwoodChannels logger = Redwood.channels(PTBLexer.class);
@@ -756,11 +756,10 @@ ABCOMP2 = Invt|Elec|Natl|M[ft]g|Dept|Blvd|Rd|Ave|[P][l]|viz
 /* ABRREV2 abbreviations are normally followed by an upper case word.
  *  We assume they aren't used sentence finally. Ph is in there for Ph. D  Sc for B.Sc.
  */
-ABBREV4 = {ABTITLE}|vs|[v]|Alex|Wm|Jos|Cie|a\.k\.a|cf|TREAS|Ph|[S][c]|{ACRO}|{ABCOMP2}
+ABBREV4 = {ABTITLE}|vs|[v]|Wm|Jos|Cie|a\.k\.a|cf|TREAS|Ph|[S][c]|{ACRO}|{ABCOMP2}
 ABBREV2 = {ABBREV4}\.
 ACRONYM = ({ACRO})\.
 /* Cie. is used by French companies sometimes before and sometimes at end as in English Co.  But we treat as allowed to have Capital following without being sentence end.  Cia. is used in Spanish/South American company abbreviations, which come before the company name, but we exclude that and lose, because in a caseless segmenter, it's too confusable with CIA. */
-/* in the WSJ Alex. is generally an abbreviation for Alex. Brown, brokers! */
 /* Added Wm. for William and Jos. for Joseph */
 /* In tables: Mkt. for market Div. for division of company, Chg., Yr.: year */
 
@@ -873,6 +872,7 @@ CP1252_MISC_SYMBOL = [\u0086\u0087\u0089\u0095\u0098\u0099]
                           if (normalizeSpace) {
                             txt = SINGLE_SPACE_PATTERN.matcher(txt).replaceAll("\u00A0"); // change to non-breaking space
                           }
+                          if (DEBUG) { logger.info("Used {SGML1} to recognize " + origTxt + " as " + txt); }
                           return getNext(txt, origTxt);
                         }
 <YyTokenizePerLine>{SGML2}
@@ -881,6 +881,7 @@ CP1252_MISC_SYMBOL = [\u0086\u0087\u0089\u0095\u0098\u0099]
                           if (normalizeSpace) {
                             txt = txt.replace(' ', '\u00A0'); // change space to non-breaking space
                           }
+                          if (DEBUG) { logger.info("Used {SGML2} to recognize " + origTxt + " as " + txt); }
                           return getNext(txt, origTxt);
                         }
 {SPMDASH}               { if (ptb3Dashes) {
@@ -970,12 +971,16 @@ CP1252_MISC_SYMBOL = [\u0086\u0087\u0089\u0095\u0098\u0099]
                                                    "; probablyLeft=" + false); }
                           return getNext(norm, tok);
                         }
-{DATE}                  { String txt = yytext();
+{DATE}                  { String origTxt = yytext();
+                          String txt;
                           if (escapeForwardSlashAsterisk) {
-                            txt = LexerUtils.escapeChar(txt, '/');
+                            txt = LexerUtils.escapeChar(origTxt, '/');
+                          } else {
+                            txt = origTxt;
                           }
-                          return getNext(txt, yytext());
-                         }
+                          if (DEBUG) { logger.info("Used {DATE} to recognize " + origTxt + " as " + txt); }
+                          return getNext(txt, origTxt);
+                        }
 /* Malaysian currency */
 RM/{NUM}        { String txt = yytext();
                   return getNext(txt, txt);
@@ -1073,8 +1078,24 @@ RM/{NUM}        { String txt = yytext();
                           // since the last one matches two things, even newlines (if not tokenize per line)
                           return processAbbrev1();
                         }
-{ABBREV2}               { return getNext(); }
-{ABBREV4}/{SPACE}       { return getNext(); }
+{ABBREV2}               { String tok = yytext();
+                          if (DEBUG) { logger.info("Used {ABBREV2} to recognize " + tok); }
+                          return getNext(tok, tok);
+                        }
+/* Last millenium (in the WSJ) "Alex." is generally an abbreviation for Alex. Brown, brokers! Recognize just this case. */
+<YyNotTokenizePerLine>Alex\./{SPACENL}Brown   { String tok = yytext();
+                                                if (DEBUG) { logger.info("Used {ALEX} to recognize " + tok); }
+                                                return getNext(tok, tok);
+                                              }
+
+<YyTokenizePerLine>Alex\./{SPACE}Brown        { String tok = yytext();
+                                                if (DEBUG) { logger.info("Used {ALEX} (2) to recognize " + tok); }
+                                                return getNext(tok, tok);
+                                              }
+{ABBREV4}/{SPACE}       { String tok = yytext();
+                          if (DEBUG) { logger.info("Used {ABBREV4} to recognize " + tok); }
+                          return getNext(tok, tok);
+                        }
 {ACRO}/{SPACENL}        { return getNext(); }
 {TBSPEC2}/{SPACENL}     { return getNext(); }
 {ISO8601DATETIME}       { return getNext(); }
@@ -1118,9 +1139,10 @@ RM/{NUM}        { String txt = yytext();
                     txt = LEFT_PAREN_PATTERN.matcher(txt).replaceAll(openparen);
                     txt = RIGHT_PAREN_PATTERN.matcher(txt).replaceAll(closeparen);
                   }
+                  if (DEBUG) { logger.info("Used {SMILEY} to recognize " + origText + " as " + txt); }
                   return getNext(txt, origText);
                 }
-{ASIANSMILEY}        { String txt = yytext();
+{ASIANSMILEY}   { String txt = yytext();
                   String origText = txt;
                   if (normalizeParentheses) {
                     txt = LEFT_PAREN_PATTERN.matcher(txt).replaceAll(openparen);