Merge pull request #1529 from h2database/lucene

Lucene upgrade to 5.5.5

h2/pom.xml

@@ -56,7 +56,17 @@
     <dependency>
       <groupId>org.apache.lucene</groupId>
       <artifactId>lucene-core</artifactId>
-      <version>3.6.2</version>
+      <version>5.5.5</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-analyzers-common</artifactId>
+      <version>5.5.5</version>
+    </dependency>
+    <dependency>
+        <groupId>org.apache.lucene</groupId>
+        <artifactId>lucene-queryparser</artifactId>
+        <version>5.5.5</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>

h2/src/docsrc/html/build.html

@@ -115,8 +115,8 @@ build jarClient
 
 <h3>Using Apache Lucene</h3>
 <p>
-Apache Lucene 3.6.2 is used for testing.
-Newer versions may work, however they are not tested.
+Apache Lucene 5.5.5 is used for testing.
+Newer versions may work, 7.5.0 was tested with recompiled under JDK 8 version.
 </p>
 
 <h2 id="maven2">Using Maven 2</h2>

h2/src/docsrc/html/tutorial.html

@@ -1234,8 +1234,8 @@ org.h2.fulltext.FullText.searchData(conn, text, limit, offset);
 <h3>Using the Apache Lucene Fulltext Search</h3>
 <p>
 To use the Apache Lucene full text search, you need the Lucene library in the classpath.
-Currently, Apache Lucene 3.6.2 is used for testing.
-Newer versions may work, however they are not tested.
+Currently, Apache Lucene 5.5.5 is used for testing.
+Newer versions may work, 7.5.0 was tested with recompiled under JDK 8 version.
 How to do that depends on the application; if you use the H2 Console, you can add the Lucene
 jar file to the environment variables <code>H2DRIVERS</code> or
 <code>CLASSPATH</code>.

h2/src/main/META-INF/MANIFEST.MF

@@ -41,14 +41,14 @@ Import-Package: javax.crypto,
  org.w3c.dom;resolution:=optional,
  org.xml.sax;resolution:=optional,
  org.xml.sax.helpers;resolution:=optional,
- org.apache.lucene.analysis;version="[3.6.2,4.0.0)";resolution:=optional,
- org.apache.lucene.analysis.standard;version="[3.6.2,4.0.0)";resolution:=optional,
- org.apache.lucene.document;version="[3.6.2,4.0.0)";resolution:=optional,
- org.apache.lucene.index;version="[3.6.2,4.0.0)";resolution:=optional,
- org.apache.lucene.queryParser;version="[3.6.2,4.0.0)";resolution:=optional,
- org.apache.lucene.search;version="[3.6.2,4.0.0)";resolution:=optional,
- org.apache.lucene.store;version="[3.6.2,4.0.0)";resolution:=optional,
- org.apache.lucene.util;version="[3.6.2,4.0.0)";resolution:=optional,
+ org.apache.lucene.analysis;version="[5.5.5,6.0.0)";resolution:=optional,
+ org.apache.lucene.analysis.standard;version="[5.5.5,6.0.0)";resolution:=optional,
+ org.apache.lucene.document;version="[5.5.5,6.0.0)";resolution:=optional,
+ org.apache.lucene.index;version="[5.5.5,6.0.0)";resolution:=optional,
+ org.apache.lucene.queryparser;version="[5.5.5,6.0.0)";resolution:=optional,
+ org.apache.lucene.search;version="[5.5.5,6.0.0)";resolution:=optional,
+ org.apache.lucene.store;version="[5.5.5,6.0.0)";resolution:=optional,
+ org.apache.lucene.util;version="[5.5.5,6.0.0)";resolution:=optional,
  org.locationtech.jts.geom;version="1.15.0";resolution:=optional,
  org.locationtech.jts.io;version="1.15.0";resolution:=optional,
  org.osgi.framework;version="1.5",

h2/src/main/org/h2/fulltext/FullTextLucene.java

@@ -5,8 +5,8 @@
  */
 package org.h2.fulltext;
 
-import java.io.File;
 import java.io.IOException;
+import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.PreparedStatement;
@@ -16,18 +16,21 @@ import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Map;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
@@ -35,7 +38,6 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
 import org.h2.api.Trigger;
 import org.h2.command.Parser;
 import org.h2.engine.Session;
@@ -293,20 +295,24 @@ public class FullTextLucene extends FullText {
         String path = getIndexPath(conn);
         synchronized (INDEX_ACCESS) {
             IndexAccess access = INDEX_ACCESS.get(path);
-            if (access == null) {
+            while (access == null) {
                 try {
                     Directory indexDir = path.startsWith(IN_MEMORY_PREFIX) ?
-                            new RAMDirectory() : FSDirectory.open(new File(path));
-                    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30);
-                    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_30, analyzer);
+                            new RAMDirectory() : FSDirectory.open(Paths.get(path));
+                    Analyzer analyzer = new StandardAnalyzer();
+                    IndexWriterConfig conf = new IndexWriterConfig(analyzer);
                     conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
                     IndexWriter writer = new IndexWriter(indexDir, conf);
                     //see http://wiki.apache.org/lucene-java/NearRealtimeSearch
                     access = new IndexAccess(writer);
+                } catch (IndexFormatTooOldException e) {
+                    reindex(conn);
+                    continue;
                 } catch (IOException e) {
                     throw convertException(e);
                 }
                 INDEX_ACCESS.put(path, access);
+                break;
             }
             return access;
         }
@@ -416,17 +422,18 @@ public class FullTextLucene extends FullText {
                 // reuse the same analyzer; it's thread-safe;
                 // also allows subclasses to control the analyzer used.
                 Analyzer analyzer = access.writer.getAnalyzer();
-                QueryParser parser = new QueryParser(Version.LUCENE_30,
-                        LUCENE_FIELD_DATA, analyzer);
-                Query query = parser.parse(text);
-                // Lucene 3 insists on a hard limit and will not provide
+                StandardQueryParser parser = new StandardQueryParser(analyzer);
+                Query query = parser.parse(text, LUCENE_FIELD_DATA);
+                // Lucene insists on a hard limit and will not provide
                 // a total hits value. Take at least 100 which is
                 // an optimal limit for Lucene as any more
                 // will trigger writing results to disk.
                 int maxResults = (limit == 0 ? 100 : limit) + offset;
                 TopDocs docs = searcher.search(query, maxResults);
                 if (limit == 0) {
-                    limit = docs.totalHits;
+                    // TopDocs.totalHits is long now (https://issues.apache.org/jira/browse/LUCENE-7872)
+                    // but in this context it's save to cast
+                    limit = (int)docs.totalHits;
                 }
                 for (int i = 0, len = docs.scoreDocs.length; i < limit
                         && i + offset < docs.totalHits
@@ -476,6 +483,14 @@ public class FullTextLucene extends FullText {
         private String indexPath;
         private IndexAccess indexAccess;
 
+        private final FieldType DOC_ID_FIELD_TYPE;
+
+        public FullTextTrigger() {
+            DOC_ID_FIELD_TYPE = new FieldType(TextField.TYPE_STORED);
+            DOC_ID_FIELD_TYPE.setTokenized(false);
+            DOC_ID_FIELD_TYPE.freeze();
+        }
+
         /**
          * INTERNAL
          */
@@ -598,12 +613,11 @@ public class FullTextLucene extends FullText {
         protected void insert(Object[] row, boolean commitIndex) throws SQLException {
             String query = getQuery(row);
             Document doc = new Document();
-            doc.add(new Field(LUCENE_FIELD_QUERY, query,
-                    Field.Store.YES, Field.Index.NOT_ANALYZED));
+            doc.add(new Field(LUCENE_FIELD_QUERY, query, DOC_ID_FIELD_TYPE));
             long time = System.currentTimeMillis();
             doc.add(new Field(LUCENE_FIELD_MODIFIED,
                     DateTools.timeToString(time, DateTools.Resolution.SECOND),
-                    Field.Store.YES, Field.Index.NOT_ANALYZED));
+                    TextField.TYPE_STORED));
             StatementBuilder buff = new StatementBuilder();
             for (int index : indexColumns) {
                 String columnName = columns[index];
@@ -614,15 +628,13 @@ public class FullTextLucene extends FullText {
                 if (columnName.startsWith(LUCENE_FIELD_COLUMN_PREFIX)) {
                     columnName = LUCENE_FIELD_COLUMN_PREFIX + columnName;
                 }
-                doc.add(new Field(columnName, data,
-                        Field.Store.NO, Field.Index.ANALYZED));
+                doc.add(new Field(columnName, data, TextField.TYPE_NOT_STORED));
                 buff.appendExceptFirst(" ");
                 buff.append(data);
             }
-            Field.Store storeText = STORE_DOCUMENT_TEXT_IN_INDEX ?
-                    Field.Store.YES : Field.Store.NO;
-            doc.add(new Field(LUCENE_FIELD_DATA, buff.toString(), storeText,
-                    Field.Index.ANALYZED));
+            FieldType dataFieldType = STORE_DOCUMENT_TEXT_IN_INDEX ?
+                    TextField.TYPE_STORED : TextField.TYPE_NOT_STORED;
+            doc.add(new Field(LUCENE_FIELD_DATA, buff.toString(), dataFieldType));
             try {
                 indexAccess.writer.addDocument(doc);
                 if (commitIndex) {
@@ -675,23 +687,13 @@ public class FullTextLucene extends FullText {
     /**
      * A wrapper for the Lucene writer and searcher.
      */
-    static final class IndexAccess {
+    private static final class IndexAccess {
 
         /**
          * The index writer.
          */
         final IndexWriter writer;
 
-        /**
-         * Map of usage counters for outstanding searchers.
-         */
-        private final Map<IndexSearcher,Integer> counters = new HashMap<>();
-
-        /**
-         * Usage counter for current searcher.
-         */
-        private int counter;
-
         /**
          * The index searcher.
          */
@@ -699,8 +701,7 @@ public class FullTextLucene extends FullText {
 
         IndexAccess(IndexWriter writer) throws IOException {
             this.writer = writer;
-            IndexReader reader = IndexReader.open(writer, true);
-            searcher = new IndexSearcher(reader);
+            initializeSearcher();
         }
 
         /**
@@ -708,29 +709,25 @@ public class FullTextLucene extends FullText {
          *
          * @return the searcher
          */
-        synchronized IndexSearcher getSearcher() {
-            ++counter;
+        synchronized IndexSearcher getSearcher() throws IOException {
+            if (!searcher.getIndexReader().tryIncRef()) {
+                initializeSearcher();
+            }
             return searcher;
         }
 
+        private void initializeSearcher() throws IOException {
+            IndexReader reader = DirectoryReader.open(writer);
+            searcher = new IndexSearcher(reader);
+        }
+
         /**
          * Stop using the searcher.
          *
          * @param searcher the searcher
          */
-        synchronized void returnSearcher(IndexSearcher searcher) {
-            if (this.searcher == searcher) {
-                --counter;
-                assert counter >= 0;
-            } else {
-                Integer cnt = counters.remove(searcher);
-                assert cnt != null;
-                if(--cnt == 0) {
-                    closeSearcher(searcher);
-                } else {
-                    counters.put(searcher, cnt);
-                }
-            }
+        synchronized void returnSearcher(IndexSearcher searcher) throws IOException {
+            searcher.getIndexReader().decRef();
         }
 
         /**
@@ -738,33 +735,16 @@ public class FullTextLucene extends FullText {
          */
         public synchronized void commit() throws IOException {
             writer.commit();
-            if (counter != 0) {
-                counters.put(searcher, counter);
-                counter = 0;
-            } else {
-                closeSearcher(searcher);
-            }
-            // recreate Searcher with the IndexWriter's reader.
-            searcher = new IndexSearcher(IndexReader.open(writer, true));
+            returnSearcher(searcher);
+            searcher = new IndexSearcher(DirectoryReader.open(writer));
         }
 
         /**
          * Close the index.
          */
         public synchronized void close() throws IOException {
-            for (IndexSearcher searcher : counters.keySet()) {
-                closeSearcher(searcher);
-            }
-            counters.clear();
-            closeSearcher(searcher);
             searcher = null;
             writer.close();
         }
-
-        private static void closeSearcher(IndexSearcher searcher) {
-            IndexReader indexReader = searcher.getIndexReader();
-            try { searcher.close(); } catch(IOException ignore) {/**/}
-            try { indexReader.close(); } catch(IOException ignore) {/**/}
-        }
     }
 }

h2/src/tools/org/h2/build/Build.java

@@ -184,7 +184,9 @@ public class Build extends BuildBase {
         String cp = "coverage/bin" +
             File.pathSeparator + "ext/postgresql-42.2.1.jre7" +
             File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
-            File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
+            File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
+            File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
+            File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
             File.pathSeparator + "ext/h2mig_pagestore_addon.jar" +
             File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
             File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
@@ -263,7 +265,9 @@ public class Build extends BuildBase {
         download();
         String classpath = "temp" +
                 File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
-                File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
+                File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
                 File.pathSeparator + "ext/slf4j-api-1.6.0.jar" +
                 File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
                 File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
@@ -358,9 +362,15 @@ public class Build extends BuildBase {
         downloadOrVerify("ext/servlet-api-3.1.0.jar",
                 "javax/servlet", "javax.servlet-api", "3.1.0",
                 "3cd63d075497751784b2fa84be59432f4905bf7c", offline);
-        downloadOrVerify("ext/lucene-core-3.6.2.jar",
-                "org/apache/lucene", "lucene-core", "3.6.2",
-                "9ec77e2507f9cc01756964c71d91efd8154a8c47", offline);
+        downloadOrVerify("ext/lucene-core-5.5.5.jar",
+                "org/apache/lucene", "lucene-core", "5.5.5",
+                "c34bcd9274859dc07cfed2a935aaca90c4f4b861", offline);
+        downloadOrVerify("ext/lucene-analyzers-common-5.5.5.jar",
+                "org/apache/lucene", "lucene-analyzers-common", "5.5.5",
+                "e6b3f5d1b33ed24da7eef0a72f8062bd4652700c", offline);
+        downloadOrVerify("ext/lucene-queryparser-5.5.5.jar",
+                "org/apache/lucene", "lucene-queryparser", "5.5.5",
+                "6c965eb5838a2ba58b0de0fd860a420dcda11937", offline);
         downloadOrVerify("ext/slf4j-api-1.6.0.jar",
                 "org/slf4j", "slf4j-api", "1.6.0",
                 "b353147a7d51fcfcd818d8aa6784839783db0915", offline);
@@ -626,7 +636,9 @@ public class Build extends BuildBase {
         javadoc("-sourcepath", "src/main", "org.h2.jdbc", "org.h2.jdbcx",
                 "org.h2.tools", "org.h2.api", "org.h2.engine", "org.h2.fulltext",
                 "-classpath",
-                "ext/lucene-core-3.6.2.jar" +
+                "ext/lucene-core-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
                 File.pathSeparator + "ext/jts-core-1.15.0.jar",
                 "-docletpath", "bin" + File.pathSeparator + "temp",
                 "-doclet", "org.h2.build.doclet.Doclet");
@@ -650,7 +662,9 @@ public class Build extends BuildBase {
                 "-classpath", javaToolsJar +
                 File.pathSeparator + "ext/slf4j-api-1.6.0.jar" +
                 File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
-                File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
+                File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
                 File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
                 File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
                 File.pathSeparator + "ext/jts-core-1.15.0.jar",
@@ -664,7 +678,9 @@ public class Build extends BuildBase {
                 "-classpath", javaToolsJar +
                 File.pathSeparator + "ext/slf4j-api-1.6.0.jar" +
                 File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
-                File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
+                File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
                 File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
                 File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
                 File.pathSeparator + "ext/jts-core-1.15.0.jar",
@@ -679,7 +695,9 @@ public class Build extends BuildBase {
                 "-classpath", javaToolsJar +
                 File.pathSeparator + "ext/slf4j-api-1.6.0.jar" +
                 File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
-                File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
+                File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
                 File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
                 File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
                 File.pathSeparator + "ext/jts-core-1.15.0.jar",
@@ -871,7 +889,6 @@ public class Build extends BuildBase {
         downloadOrVerify(true);
         if (filesMissing) {
             println("Required files are missing");
-            println("Both Lucene 2 and 3 are supported using -Dlucene=x (x=2 or 3)");
         } else {
             jar();
         }
@@ -882,7 +899,9 @@ public class Build extends BuildBase {
             java("org.h2.build.doc.GenerateHelp", null);
             javadoc("-sourcepath", "src/main", "org.h2.tools", "org.h2.jmx",
                     "-classpath",
-                    "ext/lucene-core-3.6.2.jar" +
+                    "ext/lucene-core-5.5.5.jar" +
+                    File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
+                    File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
                     File.pathSeparator + "ext/jts-core-1.15.0.jar",
                     "-docletpath", "bin" + File.pathSeparator + "temp",
                     "-doclet", "org.h2.build.doclet.ResourceDoclet");
@@ -934,7 +953,9 @@ public class Build extends BuildBase {
         String cp = "temp" + File.pathSeparator + "bin" +
                 File.pathSeparator + "ext/postgresql-42.2.1.jre7.jar" +
                 File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
-                File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
+                File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
+                File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
                 File.pathSeparator + "ext/h2mig_pagestore_addon.jar" +
                 File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
                 File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +