Skip to content

H
h2database
Unverified 提交 e03d7a25 authored 作者: Andrei Tokar's avatar Andrei Tokar 提交者： GitHub
浏览文件
操作

Merge pull request #1529 from h2database/lucene 

Lucene upgrade to 5.5.5
上级 da2fcc08 b6da6437
隐藏空白字符变更
内嵌 并排
正在显示 包含 105 行增加 和 94 行删除
h2/pom.xml
浏览文件 @ e03d7a25
......@@ -56,7 +56,17 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>3.6.2</version>
<version>5.5.5</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>5.5.5</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>5.5.5</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
......
h2/src/docsrc/html/build.html
浏览文件 @ e03d7a25
......@@ -115,8 +115,8 @@ build jarClient
<h3>Using Apache Lucene</h3>
<p>
Apache Lucene 3.6.2 is used for testing.
Newer versions may work, however they are not tested.
Apache Lucene 5.5.5 is used for testing.
Newer versions may work, 7.5.0 was tested with recompiled under JDK 8 version.
</p>
<h2 id="maven2">Using Maven 2</h2>
......
h2/src/docsrc/html/tutorial.html
浏览文件 @ e03d7a25
......@@ -1234,8 +1234,8 @@ org.h2.fulltext.FullText.searchData(conn, text, limit, offset);
<h3>Using the Apache Lucene Fulltext Search</h3>
<p>
To use the Apache Lucene full text search, you need the Lucene library in the classpath.
Currently, Apache Lucene 3.6.2 is used for testing.
Newer versions may work, however they are not tested.
Currently, Apache Lucene 5.5.5 is used for testing.
Newer versions may work, 7.5.0 was tested with recompiled under JDK 8 version.
How to do that depends on the application; if you use the H2 Console, you can add the Lucene
jar file to the environment variables <code>H2DRIVERS</code> or
<code>CLASSPATH</code>.
......
h2/src/main/META-INF/MANIFEST.MF
浏览文件 @ e03d7a25
......@@ -41,14 +41,14 @@ Import-Package: javax.crypto,
org.w3c.dom;resolution:=optional,
org.xml.sax;resolution:=optional,
org.xml.sax.helpers;resolution:=optional,
org.apache.lucene.analysis;version="[3.6.2,4.0.0)";resolution:=optional,
org.apache.lucene.analysis.standard;version="[3.6.2,4.0.0)";resolution:=optional,
org.apache.lucene.document;version="[3.6.2,4.0.0)";resolution:=optional,
org.apache.lucene.index;version="[3.6.2,4.0.0)";resolution:=optional,
org.apache.lucene.queryParser;version="[3.6.2,4.0.0)";resolution:=optional,
org.apache.lucene.search;version="[3.6.2,4.0.0)";resolution:=optional,
org.apache.lucene.store;version="[3.6.2,4.0.0)";resolution:=optional,
org.apache.lucene.util;version="[3.6.2,4.0.0)";resolution:=optional,
org.apache.lucene.analysis;version="[5.5.5,6.0.0)";resolution:=optional,
org.apache.lucene.analysis.standard;version="[5.5.5,6.0.0)";resolution:=optional,
org.apache.lucene.document;version="[5.5.5,6.0.0)";resolution:=optional,
org.apache.lucene.index;version="[5.5.5,6.0.0)";resolution:=optional,
org.apache.lucene.queryparser;version="[5.5.5,6.0.0)";resolution:=optional,
org.apache.lucene.search;version="[5.5.5,6.0.0)";resolution:=optional,
org.apache.lucene.store;version="[5.5.5,6.0.0)";resolution:=optional,
org.apache.lucene.util;version="[5.5.5,6.0.0)";resolution:=optional,
org.locationtech.jts.geom;version="1.15.0";resolution:=optional,
org.locationtech.jts.io;version="1.15.0";resolution:=optional,
org.osgi.framework;version="1.5",
......
h2/src/main/org/h2/fulltext/FullTextLucene.java
浏览文件 @ e03d7a25
......@@ -5,8 +5,8 @@
*/
package org.h2.fulltext;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
......@@ -16,18 +16,21 @@ import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
......@@ -35,7 +38,6 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.h2.api.Trigger;
import org.h2.command.Parser;
import org.h2.engine.Session;
......@@ -293,20 +295,24 @@ public class FullTextLucene extends FullText {
String path = getIndexPath(conn);
synchronized (INDEX_ACCESS) {
IndexAccess access = INDEX_ACCESS.get(path);
if (access == null) {
while (access == null) {
try {
Directory indexDir = path.startsWith(IN_MEMORY_PREFIX) ?
new RAMDirectory() : FSDirectory.open(new File(path));
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30);
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_30, analyzer);
new RAMDirectory() : FSDirectory.open(Paths.get(path));
Analyzer analyzer = new StandardAnalyzer();
IndexWriterConfig conf = new IndexWriterConfig(analyzer);
conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
IndexWriter writer = new IndexWriter(indexDir, conf);
//see http://wiki.apache.org/lucene-java/NearRealtimeSearch
access = new IndexAccess(writer);
} catch (IndexFormatTooOldException e) {
reindex(conn);
continue;
} catch (IOException e) {
throw convertException(e);
}
INDEX_ACCESS.put(path, access);
break;
}
return access;
}
......@@ -416,17 +422,18 @@ public class FullTextLucene extends FullText {
// reuse the same analyzer; it's thread-safe;
// also allows subclasses to control the analyzer used.
Analyzer analyzer = access.writer.getAnalyzer();
QueryParser parser = new QueryParser(Version.LUCENE_30,
LUCENE_FIELD_DATA, analyzer);
Query query = parser.parse(text);
// Lucene 3 insists on a hard limit and will not provide
StandardQueryParser parser = new StandardQueryParser(analyzer);
Query query = parser.parse(text, LUCENE_FIELD_DATA);
// Lucene insists on a hard limit and will not provide
// a total hits value. Take at least 100 which is
// an optimal limit for Lucene as any more
// will trigger writing results to disk.
int maxResults = (limit == 0 ? 100 : limit) + offset;
TopDocs docs = searcher.search(query, maxResults);
if (limit == 0) {
limit = docs.totalHits;
// TopDocs.totalHits is long now (https://issues.apache.org/jira/browse/LUCENE-7872)
// but in this context it's save to cast
limit = (int)docs.totalHits;
}
for (int i = 0, len = docs.scoreDocs.length; i < limit
&& i + offset < docs.totalHits
......@@ -476,6 +483,14 @@ public class FullTextLucene extends FullText {
private String indexPath;
private IndexAccess indexAccess;
private final FieldType DOC_ID_FIELD_TYPE;
public FullTextTrigger() {
DOC_ID_FIELD_TYPE = new FieldType(TextField.TYPE_STORED);
DOC_ID_FIELD_TYPE.setTokenized(false);
DOC_ID_FIELD_TYPE.freeze();
}
/**
* INTERNAL
*/
......@@ -598,12 +613,11 @@ public class FullTextLucene extends FullText {
protected void insert(Object[] row, boolean commitIndex) throws SQLException {
String query = getQuery(row);
Document doc = new Document();
doc.add(new Field(LUCENE_FIELD_QUERY, query,
Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.add(new Field(LUCENE_FIELD_QUERY, query, DOC_ID_FIELD_TYPE));
long time = System.currentTimeMillis();
doc.add(new Field(LUCENE_FIELD_MODIFIED,
DateTools.timeToString(time, DateTools.Resolution.SECOND),
Field.Store.YES, Field.Index.NOT_ANALYZED));
TextField.TYPE_STORED));
StatementBuilder buff = new StatementBuilder();
for (int index : indexColumns) {
String columnName = columns[index];
......@@ -614,15 +628,13 @@ public class FullTextLucene extends FullText {
if (columnName.startsWith(LUCENE_FIELD_COLUMN_PREFIX)) {
columnName = LUCENE_FIELD_COLUMN_PREFIX + columnName;
}
doc.add(new Field(columnName, data,
Field.Store.NO, Field.Index.ANALYZED));
doc.add(new Field(columnName, data, TextField.TYPE_NOT_STORED));
buff.appendExceptFirst(" ");
buff.append(data);
}
Field.Store storeText = STORE_DOCUMENT_TEXT_IN_INDEX ?
Field.Store.YES : Field.Store.NO;
doc.add(new Field(LUCENE_FIELD_DATA, buff.toString(), storeText,
Field.Index.ANALYZED));
FieldType dataFieldType = STORE_DOCUMENT_TEXT_IN_INDEX ?
TextField.TYPE_STORED : TextField.TYPE_NOT_STORED;
doc.add(new Field(LUCENE_FIELD_DATA, buff.toString(), dataFieldType));
try {
indexAccess.writer.addDocument(doc);
if (commitIndex) {
......@@ -675,23 +687,13 @@ public class FullTextLucene extends FullText {
/**
* A wrapper for the Lucene writer and searcher.
*/
static final class IndexAccess {
private static final class IndexAccess {
/**
* The index writer.
*/
final IndexWriter writer;
/**
* Map of usage counters for outstanding searchers.
*/
private final Map<IndexSearcher,Integer> counters = new HashMap<>();
/**
* Usage counter for current searcher.
*/
private int counter;
/**
* The index searcher.
*/
......@@ -699,8 +701,7 @@ public class FullTextLucene extends FullText {
IndexAccess(IndexWriter writer) throws IOException {
this.writer = writer;
IndexReader reader = IndexReader.open(writer, true);
searcher = new IndexSearcher(reader);
initializeSearcher();
}
/**
......@@ -708,29 +709,25 @@ public class FullTextLucene extends FullText {
*
* @return the searcher
*/
synchronized IndexSearcher getSearcher() {
++counter;
synchronized IndexSearcher getSearcher() throws IOException {
if (!searcher.getIndexReader().tryIncRef()) {
initializeSearcher();
}
return searcher;
}
private void initializeSearcher() throws IOException {
IndexReader reader = DirectoryReader.open(writer);
searcher = new IndexSearcher(reader);
}
/**
* Stop using the searcher.
*
* @param searcher the searcher
*/
synchronized void returnSearcher(IndexSearcher searcher) {
if (this.searcher == searcher) {
--counter;
assert counter >= 0;
} else {
Integer cnt = counters.remove(searcher);
assert cnt != null;
if(--cnt == 0) {
closeSearcher(searcher);
} else {
counters.put(searcher, cnt);
}
}
synchronized void returnSearcher(IndexSearcher searcher) throws IOException {
searcher.getIndexReader().decRef();
}
/**
......@@ -738,33 +735,16 @@ public class FullTextLucene extends FullText {
*/
public synchronized void commit() throws IOException {
writer.commit();
if (counter != 0) {
counters.put(searcher, counter);
counter = 0;
} else {
closeSearcher(searcher);
}
// recreate Searcher with the IndexWriter's reader.
searcher = new IndexSearcher(IndexReader.open(writer, true));
returnSearcher(searcher);
searcher = new IndexSearcher(DirectoryReader.open(writer));
}
/**
* Close the index.
*/
public synchronized void close() throws IOException {
for (IndexSearcher searcher : counters.keySet()) {
closeSearcher(searcher);
}
counters.clear();
closeSearcher(searcher);
searcher = null;
writer.close();
}
private static void closeSearcher(IndexSearcher searcher) {
IndexReader indexReader = searcher.getIndexReader();
try { searcher.close(); } catch(IOException ignore) {/**/}
try { indexReader.close(); } catch(IOException ignore) {/**/}
}
}
}
h2/src/tools/org/h2/build/Build.java
浏览文件 @ e03d7a25
......@@ -184,7 +184,9 @@ public class Build extends BuildBase {
String cp = "coverage/bin" +
File.pathSeparator + "ext/postgresql-42.2.1.jre7" +
File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
File.pathSeparator + "ext/h2mig_pagestore_addon.jar" +
File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
......@@ -263,7 +265,9 @@ public class Build extends BuildBase {
download();
String classpath = "temp" +
File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
File.pathSeparator + "ext/slf4j-api-1.6.0.jar" +
File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
......@@ -358,9 +362,15 @@ public class Build extends BuildBase {
downloadOrVerify("ext/servlet-api-3.1.0.jar",
"javax/servlet", "javax.servlet-api", "3.1.0",
"3cd63d075497751784b2fa84be59432f4905bf7c", offline);
downloadOrVerify("ext/lucene-core-3.6.2.jar",
"org/apache/lucene", "lucene-core", "3.6.2",
"9ec77e2507f9cc01756964c71d91efd8154a8c47", offline);
downloadOrVerify("ext/lucene-core-5.5.5.jar",
"org/apache/lucene", "lucene-core", "5.5.5",
"c34bcd9274859dc07cfed2a935aaca90c4f4b861", offline);
downloadOrVerify("ext/lucene-analyzers-common-5.5.5.jar",
"org/apache/lucene", "lucene-analyzers-common", "5.5.5",
"e6b3f5d1b33ed24da7eef0a72f8062bd4652700c", offline);
downloadOrVerify("ext/lucene-queryparser-5.5.5.jar",
"org/apache/lucene", "lucene-queryparser", "5.5.5",
"6c965eb5838a2ba58b0de0fd860a420dcda11937", offline);
downloadOrVerify("ext/slf4j-api-1.6.0.jar",
"org/slf4j", "slf4j-api", "1.6.0",
"b353147a7d51fcfcd818d8aa6784839783db0915", offline);
......@@ -626,7 +636,9 @@ public class Build extends BuildBase {
javadoc("-sourcepath", "src/main", "org.h2.jdbc", "org.h2.jdbcx",
"org.h2.tools", "org.h2.api", "org.h2.engine", "org.h2.fulltext",
"-classpath",
"ext/lucene-core-3.6.2.jar" +
"ext/lucene-core-5.5.5.jar" +
File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
File.pathSeparator + "ext/jts-core-1.15.0.jar",
"-docletpath", "bin" + File.pathSeparator + "temp",
"-doclet", "org.h2.build.doclet.Doclet");
......@@ -650,7 +662,9 @@ public class Build extends BuildBase {
"-classpath", javaToolsJar +
File.pathSeparator + "ext/slf4j-api-1.6.0.jar" +
File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
File.pathSeparator + "ext/jts-core-1.15.0.jar",
......@@ -664,7 +678,9 @@ public class Build extends BuildBase {
"-classpath", javaToolsJar +
File.pathSeparator + "ext/slf4j-api-1.6.0.jar" +
File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
File.pathSeparator + "ext/jts-core-1.15.0.jar",
......@@ -679,7 +695,9 @@ public class Build extends BuildBase {
"-classpath", javaToolsJar +
File.pathSeparator + "ext/slf4j-api-1.6.0.jar" +
File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
File.pathSeparator + "ext/jts-core-1.15.0.jar",
......@@ -871,7 +889,6 @@ public class Build extends BuildBase {
downloadOrVerify(true);
if (filesMissing) {
println("Required files are missing");
println("Both Lucene 2 and 3 are supported using -Dlucene=x (x=2 or 3)");
} else {
jar();
}
......@@ -882,7 +899,9 @@ public class Build extends BuildBase {
java("org.h2.build.doc.GenerateHelp", null);
javadoc("-sourcepath", "src/main", "org.h2.tools", "org.h2.jmx",
"-classpath",
"ext/lucene-core-3.6.2.jar" +
"ext/lucene-core-5.5.5.jar" +
File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
File.pathSeparator + "ext/jts-core-1.15.0.jar",
"-docletpath", "bin" + File.pathSeparator + "temp",
"-doclet", "org.h2.build.doclet.ResourceDoclet");
......@@ -934,7 +953,9 @@ public class Build extends BuildBase {
String cp = "temp" + File.pathSeparator + "bin" +
File.pathSeparator + "ext/postgresql-42.2.1.jre7.jar" +
File.pathSeparator + "ext/servlet-api-3.1.0.jar" +
File.pathSeparator + "ext/lucene-core-3.6.2.jar" +
File.pathSeparator + "ext/lucene-core-5.5.5.jar" +
File.pathSeparator + "ext/lucene-analyzers-common-5.5.5.jar" +
File.pathSeparator + "ext/lucene-queryparser-5.5.5.jar" +
File.pathSeparator + "ext/h2mig_pagestore_addon.jar" +
File.pathSeparator + "ext/org.osgi.core-4.2.0.jar" +
File.pathSeparator + "ext/org.osgi.enterprise-4.2.0.jar" +
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论