提交 5841c4df authored 作者: Noel Grandin's avatar Noel Grandin 提交者: GitHub

Merge pull request #575 from andreitokar/full_text_mt

Support for full text search in multithreaded mode
...@@ -1320,13 +1320,6 @@ SELECT * FROM FTL_SEARCH_DATA('John', 0, 0); ...@@ -1320,13 +1320,6 @@ SELECT * FROM FTL_SEARCH_DATA('John', 0, 0);
SELECT * FROM FTL_SEARCH_DATA('LAST_NAME:John', 0, 0); SELECT * FROM FTL_SEARCH_DATA('LAST_NAME:John', 0, 0);
CALL FTL_DROP_ALL(); CALL FTL_DROP_ALL();
</pre> </pre>
<p>
The Lucene fulltext search implementation is not synchronized internally.
If you update the database and query the fulltext search concurrently
(directly using the Java API of H2 or Lucene itself), you need to ensure
operations are properly synchronized. If this is not the case, you may get
exceptions such as <code>org.apache.lucene.store.AlreadyClosedException: this IndexReader is closed</code>.
</p>
<h2 id="user_defined_variables">User-Defined Variables</h2> <h2 id="user_defined_variables">User-Defined Variables</h2>
<p> <p>
......
...@@ -18,11 +18,12 @@ import java.sql.Statement; ...@@ -18,11 +18,12 @@ import java.sql.Statement;
import java.sql.Types; import java.sql.Types;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Set;
import java.util.StringTokenizer; import java.util.StringTokenizer;
import java.util.UUID; import java.util.UUID;
import org.h2.api.Trigger; import org.h2.api.Trigger;
import org.h2.command.Parser; import org.h2.command.Parser;
import org.h2.engine.Session; import org.h2.engine.Session;
...@@ -151,13 +152,12 @@ public class FullText { ...@@ -151,13 +152,12 @@ public class FullText {
} }
} }
rs = stat.executeQuery("SELECT * FROM " + SCHEMA + ".WORDS"); rs = stat.executeQuery("SELECT * FROM " + SCHEMA + ".WORDS");
HashMap<String, Integer> map = setting.getWordList();
while (rs.next()) { while (rs.next()) {
String word = rs.getString("NAME"); String word = rs.getString("NAME");
int id = rs.getInt("ID"); int id = rs.getInt("ID");
word = setting.convertWord(word); word = setting.convertWord(word);
if (word != null) { if (word != null) {
map.put(word, id); setting.addWord(word, id);
} }
} }
setting.setInitialized(true); setting.setInitialized(true);
...@@ -195,7 +195,7 @@ public class FullText { ...@@ -195,7 +195,7 @@ public class FullText {
init(conn); init(conn);
removeAllTriggers(conn, TRIGGER_PREFIX); removeAllTriggers(conn, TRIGGER_PREFIX);
FullTextSettings setting = FullTextSettings.getInstance(conn); FullTextSettings setting = FullTextSettings.getInstance(conn);
setting.getWordList().clear(); setting.clearWordList();
Statement stat = conn.createStatement(); Statement stat = conn.createStatement();
stat.execute("TRUNCATE TABLE " + SCHEMA + ".WORDS"); stat.execute("TRUNCATE TABLE " + SCHEMA + ".WORDS");
stat.execute("TRUNCATE TABLE " + SCHEMA + ".ROWS"); stat.execute("TRUNCATE TABLE " + SCHEMA + ".ROWS");
...@@ -243,9 +243,9 @@ public class FullText { ...@@ -243,9 +243,9 @@ public class FullText {
break; break;
} }
} }
prep = conn.prepareStatement("DELETE FROM " + SCHEMA + ".MAP M " + prep = conn.prepareStatement("DELETE FROM " + SCHEMA + ".MAP " +
"WHERE NOT EXISTS (SELECT * FROM " + SCHEMA + "WHERE NOT EXISTS (SELECT * FROM " + SCHEMA +
".ROWS R WHERE R.ID=M.ROWID) AND ROWID<10000"); ".ROWS R WHERE R.ID=ROWID) AND ROWID<10000");
while (true) { while (true) {
int deleted = prep.executeUpdate(); int deleted = prep.executeUpdate();
if (deleted == 0) { if (deleted == 0) {
...@@ -266,8 +266,8 @@ public class FullText { ...@@ -266,8 +266,8 @@ public class FullText {
removeAllTriggers(conn, TRIGGER_PREFIX); removeAllTriggers(conn, TRIGGER_PREFIX);
FullTextSettings setting = FullTextSettings.getInstance(conn); FullTextSettings setting = FullTextSettings.getInstance(conn);
setting.removeAllIndexes(); setting.removeAllIndexes();
setting.getIgnoreList().clear(); setting.clearInored();
setting.getWordList().clear(); setting.clearWordList();
} }
/** /**
...@@ -605,17 +605,16 @@ public class FullText { ...@@ -605,17 +605,16 @@ public class FullText {
if (!setting.isInitialized()) { if (!setting.isInitialized()) {
init(conn); init(conn);
} }
HashSet<String> words = New.hashSet(); Set<String> words = New.hashSet();
addWords(setting, words, text); addWords(setting, words, text);
HashSet<Integer> rIds = null, lastRowIds = null; Set<Integer> rIds = null, lastRowIds;
HashMap<String, Integer> allWords = setting.getWordList();
PreparedStatement prepSelectMapByWordId = setting.prepare(conn, PreparedStatement prepSelectMapByWordId = setting.prepare(conn,
SELECT_MAP_BY_WORD_ID); SELECT_MAP_BY_WORD_ID);
for (String word : words) { for (String word : words) {
lastRowIds = rIds; lastRowIds = rIds;
rIds = New.hashSet(); rIds = New.hashSet();
Integer wId = allWords.get(word); Integer wId = setting.getWordId(word);
if (wId == null) { if (wId == null) {
continue; continue;
} }
...@@ -698,7 +697,7 @@ public class FullText { ...@@ -698,7 +697,7 @@ public class FullText {
* @param reader the reader * @param reader the reader
*/ */
protected static void addWords(FullTextSettings setting, protected static void addWords(FullTextSettings setting,
HashSet<String> set, Reader reader) { Set<String> set, Reader reader) {
StreamTokenizer tokenizer = new StreamTokenizer(reader); StreamTokenizer tokenizer = new StreamTokenizer(reader);
tokenizer.resetSyntax(); tokenizer.resetSyntax();
tokenizer.wordChars(' ' + 1, 255); tokenizer.wordChars(' ' + 1, 255);
...@@ -732,7 +731,7 @@ public class FullText { ...@@ -732,7 +731,7 @@ public class FullText {
* @param text the text * @param text the text
*/ */
protected static void addWords(FullTextSettings setting, protected static void addWords(FullTextSettings setting,
HashSet<String> set, String text) { Set<String> set, String text) {
String whitespaceChars = setting.getWhitespaceChars(); String whitespaceChars = setting.getWhitespaceChars();
StringTokenizer tokenizer = new StringTokenizer(text, whitespaceChars); StringTokenizer tokenizer = new StringTokenizer(text, whitespaceChars);
while (tokenizer.hasMoreTokens()) { while (tokenizer.hasMoreTokens()) {
...@@ -751,30 +750,38 @@ public class FullText { ...@@ -751,30 +750,38 @@ public class FullText {
* @param schema the schema name * @param schema the schema name
* @param table the table name * @param table the table name
*/ */
protected static void createTrigger(Connection conn, String schema, private static void createTrigger(Connection conn, String schema,
String table) throws SQLException { String table) throws SQLException {
createOrDropTrigger(conn, schema, table, true); createOrDropTrigger(conn, schema, table, true);
} }
private static void createOrDropTrigger(Connection conn, private static void createOrDropTrigger(Connection conn,
String schema, String table, boolean create) throws SQLException { String schema, String table, boolean create) throws SQLException {
Statement stat = conn.createStatement(); try (Statement stat = conn.createStatement()) {
String trigger = StringUtils.quoteIdentifier(schema) + "." String trigger = StringUtils.quoteIdentifier(schema) + "."
+ StringUtils.quoteIdentifier(TRIGGER_PREFIX + table); + StringUtils.quoteIdentifier(TRIGGER_PREFIX + table);
stat.execute("DROP TRIGGER IF EXISTS " + trigger); stat.execute("DROP TRIGGER IF EXISTS " + trigger);
if (create) { if (create) {
StringBuilder buff = new StringBuilder("CREATE TRIGGER IF NOT EXISTS "); boolean multiThread = FullTextTrigger.isMultiThread(conn);
// needs to be called on rollback as well, because we use the init StringBuilder buff = new StringBuilder(
// connection do to changes in the index (not the user connection) "CREATE TRIGGER IF NOT EXISTS ");
buff.append(trigger). // unless multithread, trigger needs to be called on rollback as well,
append(" AFTER INSERT, UPDATE, DELETE, ROLLBACK ON "). // because we use the init connection do to changes in the index
append(StringUtils.quoteIdentifier(schema)). // (not the user connection)
append('.'). buff.append(trigger).
append(StringUtils.quoteIdentifier(table)). append(" AFTER INSERT, UPDATE, DELETE");
append(" FOR EACH ROW CALL \""). if(!multiThread) {
append(FullText.FullTextTrigger.class.getName()). buff.append(", ROLLBACK");
append('\"'); }
stat.execute(buff.toString()); buff.append(" ON ").
append(StringUtils.quoteIdentifier(schema)).
append('.').
append(StringUtils.quoteIdentifier(table)).
append(" FOR EACH ROW CALL \"").
append(FullText.FullTextTrigger.class.getName()).
append('\"');
stat.execute(buff.toString());
}
} }
} }
...@@ -785,8 +792,8 @@ public class FullText { ...@@ -785,8 +792,8 @@ public class FullText {
* @param schema the schema name * @param schema the schema name
* @param table the table name * @param table the table name
*/ */
protected static void indexExistingRows(Connection conn, String schema, private static void indexExistingRows(Connection conn, String schema,
String table) throws SQLException { String table) throws SQLException {
FullText.FullTextTrigger existing = new FullText.FullTextTrigger(); FullText.FullTextTrigger existing = new FullText.FullTextTrigger();
existing.init(conn, schema, null, table, false, Trigger.INSERT); existing.init(conn, schema, null, table, false, Trigger.INSERT);
String sql = "SELECT * FROM " + StringUtils.quoteIdentifier(schema) + String sql = "SELECT * FROM " + StringUtils.quoteIdentifier(schema) +
...@@ -823,13 +830,7 @@ public class FullText { ...@@ -823,13 +830,7 @@ public class FullText {
private static void setIgnoreList(FullTextSettings setting, private static void setIgnoreList(FullTextSettings setting,
String commaSeparatedList) { String commaSeparatedList) {
String[] list = StringUtils.arraySplit(commaSeparatedList, ',', true); String[] list = StringUtils.arraySplit(commaSeparatedList, ',', true);
HashSet<String> set = setting.getIgnoreList(); setting.addIgnored(Arrays.asList(list));
for (String word : list) {
String converted = setting.convertWord(word);
if (converted != null) {
set.add(converted);
}
}
} }
/** /**
...@@ -860,14 +861,28 @@ public class FullText { ...@@ -860,14 +861,28 @@ public class FullText {
/** /**
* Trigger updates the index when a inserting, updating, or deleting a row. * Trigger updates the index when a inserting, updating, or deleting a row.
*/ */
public static class FullTextTrigger implements Trigger { public static final class FullTextTrigger implements Trigger {
private FullTextSettings setting;
private IndexInfo index;
private int[] columnTypes;
private final PreparedStatement[] prepStatements = new PreparedStatement[SQL.length];
private boolean useOwnConnection;
private static final int INSERT_WORD = 0;
private static final int INSERT_ROW = 1;
private static final int INSERT_MAP = 2;
private static final int DELETE_ROW = 3;
private static final int DELETE_MAP = 4;
private static final int SELECT_ROW = 5;
protected FullTextSettings setting; private static final String SQL[] = {
protected IndexInfo index; "MERGE INTO " + SCHEMA + ".WORDS(NAME) KEY(NAME) VALUES(?)",
protected int[] columnTypes; "INSERT INTO " + SCHEMA + ".ROWS(HASH, INDEXID, KEY) VALUES(?, ?, ?)",
protected PreparedStatement prepInsertWord, prepInsertRow, prepInsertMap; "INSERT INTO " + SCHEMA + ".MAP(ROWID, WORDID) VALUES(?, ?)",
protected PreparedStatement prepDeleteRow, prepDeleteMap; "DELETE FROM " + SCHEMA + ".ROWS WHERE HASH=? AND INDEXID=? AND KEY=?",
protected PreparedStatement prepSelectRow; "DELETE FROM " + SCHEMA + ".MAP WHERE ROWID=? AND WORDID=?",
"SELECT ID FROM " + SCHEMA + ".ROWS WHERE HASH=? AND INDEXID=? AND KEY=?"
};
/** /**
* INTERNAL * INTERNAL
...@@ -915,7 +930,8 @@ public class FullText { ...@@ -915,7 +930,8 @@ public class FullText {
} }
ArrayList<String> indexList = New.arrayList(); ArrayList<String> indexList = New.arrayList();
PreparedStatement prep = conn.prepareStatement( PreparedStatement prep = conn.prepareStatement(
"SELECT ID, COLUMNS FROM " + SCHEMA + ".INDEXES WHERE SCHEMA=? AND TABLE=?"); "SELECT ID, COLUMNS FROM " + SCHEMA + ".INDEXES" +
" WHERE SCHEMA=? AND TABLE=?");
prep.setString(1, schemaName); prep.setString(1, schemaName);
prep.setString(2, tableName); prep.setString(2, tableName);
rs = prep.executeQuery(); rs = prep.executeQuery();
...@@ -923,9 +939,7 @@ public class FullText { ...@@ -923,9 +939,7 @@ public class FullText {
index.id = rs.getInt(1); index.id = rs.getInt(1);
String columns = rs.getString(2); String columns = rs.getString(2);
if (columns != null) { if (columns != null) {
for (String s : StringUtils.arraySplit(columns, ',', true)) { Collections.addAll(indexList, StringUtils.arraySplit(columns, ',', true));
indexList.add(s);
}
} }
} }
if (indexList.size() == 0) { if (indexList.size() == 0) {
...@@ -936,18 +950,23 @@ public class FullText { ...@@ -936,18 +950,23 @@ public class FullText {
index.indexColumns = new int[indexList.size()]; index.indexColumns = new int[indexList.size()];
setColumns(index.indexColumns, indexList, columnList); setColumns(index.indexColumns, indexList, columnList);
setting.addIndexInfo(index); setting.addIndexInfo(index);
prepInsertWord = conn.prepareStatement(
"INSERT INTO " + SCHEMA + ".WORDS(NAME) VALUES(?)"); useOwnConnection = isMultiThread(conn);
prepInsertRow = conn.prepareStatement( if(!useOwnConnection) {
"INSERT INTO " + SCHEMA + ".ROWS(HASH, INDEXID, KEY) VALUES(?, ?, ?)"); for (int i = 0; i < SQL.length; i++) {
prepInsertMap = conn.prepareStatement( prepStatements[i] = conn.prepareStatement(SQL[i]);
"INSERT INTO " + SCHEMA + ".MAP(ROWID, WORDID) VALUES(?, ?)"); }
prepDeleteRow = conn.prepareStatement( }
"DELETE FROM " + SCHEMA + ".ROWS WHERE HASH=? AND INDEXID=? AND KEY=?"); }
prepDeleteMap = conn.prepareStatement(
"DELETE FROM " + SCHEMA + ".MAP WHERE ROWID=? AND WORDID=?"); private static boolean isMultiThread(Connection conn)
prepSelectRow = conn.prepareStatement( throws SQLException {
"SELECT ID FROM " + SCHEMA + ".ROWS WHERE HASH=? AND INDEXID=? AND KEY=?"); try (Statement stat = conn.createStatement()) {
ResultSet rs = stat.executeQuery(
"SELECT value FROM information_schema.settings" +
" WHERE name = 'MULTI_THREADED'");
return rs.next() && !"0".equals(rs.getString(1));
}
} }
/** /**
...@@ -960,16 +979,16 @@ public class FullText { ...@@ -960,16 +979,16 @@ public class FullText {
if (newRow != null) { if (newRow != null) {
// update // update
if (hasChanged(oldRow, newRow, index.indexColumns)) { if (hasChanged(oldRow, newRow, index.indexColumns)) {
delete(oldRow); delete(conn, oldRow);
insert(newRow); insert(conn, newRow);
} }
} else { } else {
// delete // delete
delete(oldRow); delete(conn, oldRow);
} }
} else if (newRow != null) { } else if (newRow != null) {
// insert // insert
insert(newRow); insert(conn, newRow);
} }
} }
...@@ -992,54 +1011,82 @@ public class FullText { ...@@ -992,54 +1011,82 @@ public class FullText {
/** /**
* Add a row to the index. * Add a row to the index.
* *
* @param conn to use
* @param row the row * @param row the row
*/ */
protected void insert(Object[] row) throws SQLException { protected void insert(Connection conn, Object[] row) throws SQLException {
String key = getKey(row); PreparedStatement prepInsertRow = null;
int hash = key.hashCode(); PreparedStatement prepInsertMap = null;
prepInsertRow.setInt(1, hash); try {
prepInsertRow.setInt(2, index.id); String key = getKey(row);
prepInsertRow.setString(3, key); int hash = key.hashCode();
prepInsertRow.execute(); prepInsertRow = getStatement(conn, INSERT_ROW);
ResultSet rs = prepInsertRow.getGeneratedKeys(); prepInsertRow.setInt(1, hash);
rs.next(); prepInsertRow.setInt(2, index.id);
int rowId = rs.getInt(1); prepInsertRow.setString(3, key);
prepInsertMap.setInt(1, rowId); prepInsertRow.execute();
int[] wordIds = getWordIds(row); ResultSet rs = prepInsertRow.getGeneratedKeys();
for (int id : wordIds) { rs.next();
prepInsertMap.setInt(2, id); int rowId = rs.getInt(1);
prepInsertMap.execute();
prepInsertMap = getStatement(conn, INSERT_MAP);
prepInsertMap.setInt(1, rowId);
int[] wordIds = getWordIds(conn, row);
for (int id : wordIds) {
prepInsertMap.setInt(2, id);
prepInsertMap.execute();
}
} finally {
if (useOwnConnection) {
IOUtils.closeSilently(prepInsertRow);
IOUtils.closeSilently(prepInsertMap);
}
} }
} }
/** /**
* Delete a row from the index. * Delete a row from the index.
* *
* @param conn to use
* @param row the row * @param row the row
*/ */
protected void delete(Object[] row) throws SQLException { protected void delete(Connection conn, Object[] row) throws SQLException {
String key = getKey(row); PreparedStatement prepSelectRow = null;
int hash = key.hashCode(); PreparedStatement prepDeleteMap = null;
prepSelectRow.setInt(1, hash); PreparedStatement prepDeleteRow = null;
prepSelectRow.setInt(2, index.id); try {
prepSelectRow.setString(3, key); String key = getKey(row);
ResultSet rs = prepSelectRow.executeQuery(); int hash = key.hashCode();
if (rs.next()) { prepSelectRow = getStatement(conn, SELECT_ROW);
int rowId = rs.getInt(1); prepSelectRow.setInt(1, hash);
prepDeleteMap.setInt(1, rowId); prepSelectRow.setInt(2, index.id);
int[] wordIds = getWordIds(row); prepSelectRow.setString(3, key);
for (int id : wordIds) { ResultSet rs = prepSelectRow.executeQuery();
prepDeleteMap.setInt(2, id); prepDeleteMap = getStatement(conn, DELETE_MAP);
prepDeleteMap.executeUpdate(); prepDeleteRow = getStatement(conn, DELETE_ROW);
if (rs.next()) {
int rowId = rs.getInt(1);
prepDeleteMap.setInt(1, rowId);
int[] wordIds = getWordIds(conn, row);
for (int id : wordIds) {
prepDeleteMap.setInt(2, id);
prepDeleteMap.executeUpdate();
}
prepDeleteRow.setInt(1, hash);
prepDeleteRow.setInt(2, index.id);
prepDeleteRow.setString(3, key);
prepDeleteRow.executeUpdate();
}
} finally {
if (useOwnConnection) {
IOUtils.closeSilently(prepSelectRow);
IOUtils.closeSilently(prepDeleteMap);
IOUtils.closeSilently(prepDeleteRow);
} }
prepDeleteRow.setInt(1, hash);
prepDeleteRow.setInt(2, index.id);
prepDeleteRow.setString(3, key);
prepDeleteRow.executeUpdate();
} }
} }
private int[] getWordIds(Object[] row) throws SQLException { private int[] getWordIds(Connection conn, Object[] row) throws SQLException {
HashSet<String> words = New.hashSet(); HashSet<String> words = New.hashSet();
for (int idx : index.indexColumns) { for (int idx : index.indexColumns) {
int type = columnTypes[idx]; int type = columnTypes[idx];
...@@ -1057,27 +1104,36 @@ public class FullText { ...@@ -1057,27 +1104,36 @@ public class FullText {
addWords(setting, words, string); addWords(setting, words, string);
} }
} }
HashMap<String, Integer> allWords = setting.getWordList(); PreparedStatement prepInsertWord = null;
int[] wordIds = new int[words.size()]; try {
Iterator<String> it = words.iterator(); prepInsertWord = getStatement(conn, INSERT_WORD);
for (int i = 0; it.hasNext(); i++) { int[] wordIds = new int[words.size()];
String word = it.next(); int i = 0;
Integer wId = allWords.get(word); for (String word : words) {
int wordId; int wordId;
if (wId == null) { Integer wId;
prepInsertWord.setString(1, word); while((wId = setting.getWordId(word)) == null) {
prepInsertWord.execute(); prepInsertWord.setString(1, word);
ResultSet rs = prepInsertWord.getGeneratedKeys(); prepInsertWord.execute();
rs.next(); ResultSet rs = prepInsertWord.getGeneratedKeys();
wordId = rs.getInt(1); if (rs.next()) {
allWords.put(word, wordId); wordId = rs.getInt(1);
} else { if (wordId != 0) {
wordId = wId.intValue(); setting.addWord(word, wordId);
wId = wordId;
break;
}
}
}
wordIds[i++] = wId;
}
Arrays.sort(wordIds);
return wordIds;
} finally {
if (useOwnConnection) {
IOUtils.closeSilently(prepInsertWord);
} }
wordIds[i] = wordId;
} }
Arrays.sort(wordIds);
return wordIds;
} }
private String getKey(Object[] row) throws SQLException { private String getKey(Object[] row) throws SQLException {
...@@ -1095,6 +1151,10 @@ public class FullText { ...@@ -1095,6 +1151,10 @@ public class FullText {
return buff.toString(); return buff.toString();
} }
private PreparedStatement getStatement(Connection conn, int indx) throws SQLException {
return useOwnConnection ? conn.prepareStatement(SQL[indx]) : prepStatements[indx];
}
} }
/** /**
...@@ -1116,5 +1176,4 @@ public class FullText { ...@@ -1116,5 +1176,4 @@ public class FullText {
throws SQLException { throws SQLException {
throw new SQLException(message, "FULLTEXT"); throw new SQLException(message, "FULLTEXT");
} }
} }
...@@ -13,6 +13,7 @@ import java.sql.ResultSet; ...@@ -13,6 +13,7 @@ import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer;
...@@ -37,6 +38,8 @@ import org.h2.util.StatementBuilder; ...@@ -37,6 +38,8 @@ import org.h2.util.StatementBuilder;
import org.h2.util.StringUtils; import org.h2.util.StringUtils;
import org.h2.util.Utils; import org.h2.util.Utils;
import java.io.File; import java.io.File;
import java.util.Map;
import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FSDirectory;
...@@ -94,27 +97,23 @@ public class FullTextLucene extends FullText { ...@@ -94,27 +97,23 @@ public class FullTextLucene extends FullText {
* @param conn the connection * @param conn the connection
*/ */
public static void init(Connection conn) throws SQLException { public static void init(Connection conn) throws SQLException {
Statement stat = conn.createStatement(); try (Statement stat = conn.createStatement()) {
stat.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); stat.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA);
stat.execute("CREATE TABLE IF NOT EXISTS " + SCHEMA + stat.execute("CREATE TABLE IF NOT EXISTS " + SCHEMA +
".INDEXES(SCHEMA VARCHAR, TABLE VARCHAR, " + ".INDEXES(SCHEMA VARCHAR, TABLE VARCHAR, " +
"COLUMNS VARCHAR, PRIMARY KEY(SCHEMA, TABLE))"); "COLUMNS VARCHAR, PRIMARY KEY(SCHEMA, TABLE))");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_CREATE_INDEX FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_CREATE_INDEX FOR \"" +
FullTextLucene.class.getName() + ".createIndex\""); FullTextLucene.class.getName() + ".createIndex\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_DROP_INDEX FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_DROP_INDEX FOR \"" +
FullTextLucene.class.getName() + ".dropIndex\""); FullTextLucene.class.getName() + ".dropIndex\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_SEARCH FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_SEARCH FOR \"" +
FullTextLucene.class.getName() + ".search\""); FullTextLucene.class.getName() + ".search\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_SEARCH_DATA FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_SEARCH_DATA FOR \"" +
FullTextLucene.class.getName() + ".searchData\""); FullTextLucene.class.getName() + ".searchData\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_REINDEX FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_REINDEX FOR \"" +
FullTextLucene.class.getName() + ".reindex\""); FullTextLucene.class.getName() + ".reindex\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_DROP_ALL FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_DROP_ALL FOR \"" +
FullTextLucene.class.getName() + ".dropAll\""); FullTextLucene.class.getName() + ".dropAll\"");
try {
getIndexAccess(conn);
} catch (SQLException e) {
throw convertException(e);
} }
} }
...@@ -157,11 +156,9 @@ public class FullTextLucene extends FullText { ...@@ -157,11 +156,9 @@ public class FullTextLucene extends FullText {
prep.setString(1, schema); prep.setString(1, schema);
prep.setString(2, table); prep.setString(2, table);
int rowCount = prep.executeUpdate(); int rowCount = prep.executeUpdate();
if (rowCount == 0) { if (rowCount != 0) {
return; reindex(conn);
} }
reindex(conn);
} }
/** /**
...@@ -248,10 +245,7 @@ public class FullTextLucene extends FullText { ...@@ -248,10 +245,7 @@ public class FullTextLucene extends FullText {
* @return the converted SQL exception * @return the converted SQL exception
*/ */
protected static SQLException convertException(Exception e) { protected static SQLException convertException(Exception e) {
SQLException e2 = new SQLException( return new SQLException("Error while indexing document", "FULLTEXT", e);
"Error while indexing document", "FULLTEXT");
e2.initCause(e);
return e2;
} }
/** /**
...@@ -261,8 +255,8 @@ public class FullTextLucene extends FullText { ...@@ -261,8 +255,8 @@ public class FullTextLucene extends FullText {
* @param schema the schema name * @param schema the schema name
* @param table the table name * @param table the table name
*/ */
protected static void createTrigger(Connection conn, String schema, private static void createTrigger(Connection conn, String schema,
String table) throws SQLException { String table) throws SQLException {
createOrDropTrigger(conn, schema, table, true); createOrDropTrigger(conn, schema, table, true);
} }
...@@ -309,11 +303,7 @@ public class FullTextLucene extends FullText { ...@@ -309,11 +303,7 @@ public class FullTextLucene extends FullText {
conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
IndexWriter writer = new IndexWriter(indexDir, conf); IndexWriter writer = new IndexWriter(indexDir, conf);
//see http://wiki.apache.org/lucene-java/NearRealtimeSearch //see http://wiki.apache.org/lucene-java/NearRealtimeSearch
IndexReader reader = IndexReader.open(writer, true); access = new IndexAccess(writer);
access = new IndexAccess();
access.writer = writer;
access.reader = reader;
access.searcher = new IndexSearcher(reader);
} catch (IOException e) { } catch (IOException e) {
throw convertException(e); throw convertException(e);
} }
...@@ -353,8 +343,8 @@ public class FullTextLucene extends FullText { ...@@ -353,8 +343,8 @@ public class FullTextLucene extends FullText {
* @param schema the schema name * @param schema the schema name
* @param table the table name * @param table the table name
*/ */
protected static void indexExistingRows(Connection conn, String schema, private static void indexExistingRows(Connection conn, String schema,
String table) throws SQLException { String table) throws SQLException {
FullTextLucene.FullTextTrigger existing = new FullTextLucene.FullTextTrigger(); FullTextLucene.FullTextTrigger existing = new FullTextLucene.FullTextTrigger();
existing.init(conn, schema, null, table, false, Trigger.INSERT); existing.init(conn, schema, null, table, false, Trigger.INSERT);
String sql = "SELECT * FROM " + StringUtils.quoteIdentifier(schema) + String sql = "SELECT * FROM " + StringUtils.quoteIdentifier(schema) +
...@@ -373,10 +363,7 @@ public class FullTextLucene extends FullText { ...@@ -373,10 +363,7 @@ public class FullTextLucene extends FullText {
private static void removeIndexFiles(Connection conn) throws SQLException { private static void removeIndexFiles(Connection conn) throws SQLException {
String path = getIndexPath(conn); String path = getIndexPath(conn);
IndexAccess access = INDEX_ACCESS.get(path); removeIndexAccess(path);
if (access != null) {
removeIndexAccess(access, path);
}
if (!path.startsWith(IN_MEMORY_PREFIX)) { if (!path.startsWith(IN_MEMORY_PREFIX)) {
FileUtils.deleteRecursive(path, false); FileUtils.deleteRecursive(path, false);
} }
...@@ -386,17 +373,16 @@ public class FullTextLucene extends FullText { ...@@ -386,17 +373,16 @@ public class FullTextLucene extends FullText {
* Close the index writer and searcher and remove them from the index access * Close the index writer and searcher and remove them from the index access
* set. * set.
* *
* @param access the index writer/searcher wrapper
* @param indexPath the index path * @param indexPath the index path
*/ */
protected static void removeIndexAccess(IndexAccess access, String indexPath) protected static void removeIndexAccess(String indexPath)
throws SQLException { throws SQLException {
synchronized (INDEX_ACCESS) { synchronized (INDEX_ACCESS) {
try { try {
INDEX_ACCESS.remove(indexPath); IndexAccess access = INDEX_ACCESS.remove(indexPath);
access.searcher.close(); if(access != null) {
access.reader.close(); access.close();
access.writer.close(); }
} catch (Exception e) { } catch (Exception e) {
throw convertException(e); throw convertException(e);
} }
...@@ -426,49 +412,53 @@ public class FullTextLucene extends FullText { ...@@ -426,49 +412,53 @@ public class FullTextLucene extends FullText {
try { try {
IndexAccess access = getIndexAccess(conn); IndexAccess access = getIndexAccess(conn);
// take a reference as the searcher may change // take a reference as the searcher may change
IndexSearcher searcher = access.searcher; IndexSearcher searcher = access.getSearcher();
// reuse the same analyzer; it's thread-safe; try {
// also allows subclasses to control the analyzer used. // reuse the same analyzer; it's thread-safe;
Analyzer analyzer = access.writer.getAnalyzer(); // also allows subclasses to control the analyzer used.
QueryParser parser = new QueryParser(Version.LUCENE_30, Analyzer analyzer = access.writer.getAnalyzer();
LUCENE_FIELD_DATA, analyzer); QueryParser parser = new QueryParser(Version.LUCENE_30,
Query query = parser.parse(text); LUCENE_FIELD_DATA, analyzer);
// Lucene 3 insists on a hard limit and will not provide Query query = parser.parse(text);
// a total hits value. Take at least 100 which is // Lucene 3 insists on a hard limit and will not provide
// an optimal limit for Lucene as any more // a total hits value. Take at least 100 which is
// will trigger writing results to disk. // an optimal limit for Lucene as any more
int maxResults = (limit == 0 ? 100 : limit) + offset; // will trigger writing results to disk.
TopDocs docs = searcher.search(query, maxResults); int maxResults = (limit == 0 ? 100 : limit) + offset;
if (limit == 0) { TopDocs docs = searcher.search(query, maxResults);
limit = docs.totalHits; if (limit == 0) {
} limit = docs.totalHits;
for (int i = 0, len = docs.scoreDocs.length; }
i < limit && i + offset < docs.totalHits for (int i = 0, len = docs.scoreDocs.length;
&& i + offset < len; i++) { i < limit && i + offset < docs.totalHits
ScoreDoc sd = docs.scoreDocs[i + offset]; && i + offset < len; i++) {
Document doc = searcher.doc(sd.doc); ScoreDoc sd = docs.scoreDocs[i + offset];
float score = sd.score; Document doc = searcher.doc(sd.doc);
String q = doc.get(LUCENE_FIELD_QUERY); float score = sd.score;
if (data) { String q = doc.get(LUCENE_FIELD_QUERY);
int idx = q.indexOf(" WHERE "); if (data) {
JdbcConnection c = (JdbcConnection) conn; int idx = q.indexOf(" WHERE ");
Session session = (Session) c.getSession(); JdbcConnection c = (JdbcConnection) conn;
Parser p = new Parser(session); Session session = (Session) c.getSession();
String tab = q.substring(0, idx); Parser p = new Parser(session);
ExpressionColumn expr = (ExpressionColumn) p.parseExpression(tab); String tab = q.substring(0, idx);
String schemaName = expr.getOriginalTableAliasName(); ExpressionColumn expr = (ExpressionColumn) p.parseExpression(tab);
String tableName = expr.getColumnName(); String schemaName = expr.getOriginalTableAliasName();
q = q.substring(idx + " WHERE ".length()); String tableName = expr.getColumnName();
Object[][] columnData = parseKey(conn, q); q = q.substring(idx + " WHERE ".length());
result.addRow( Object[][] columnData = parseKey(conn, q);
schemaName, result.addRow(
tableName, schemaName,
columnData[0], tableName,
columnData[1], columnData[0],
score); columnData[1],
} else { score);
result.addRow(q, score); } else {
result.addRow(q, score);
}
} }
} finally {
access.returnSearcher(searcher);
} }
} catch (Exception e) { } catch (Exception e) {
throw convertException(e); throw convertException(e);
...@@ -479,16 +469,16 @@ public class FullTextLucene extends FullText { ...@@ -479,16 +469,16 @@ public class FullTextLucene extends FullText {
/** /**
* Trigger updates the index when a inserting, updating, or deleting a row. * Trigger updates the index when a inserting, updating, or deleting a row.
*/ */
public static class FullTextTrigger implements Trigger { public static final class FullTextTrigger implements Trigger {
protected String schema; private String schema;
protected String table; private String table;
protected int[] keys; private int[] keys;
protected int[] indexColumns; private int[] indexColumns;
protected String[] columns; private String[] columns;
protected int[] columnTypes; private int[] columnTypes;
protected String indexPath; private String indexPath;
protected IndexAccess indexAccess; private IndexAccess indexAccess;
/** /**
* INTERNAL * INTERNAL
...@@ -541,9 +531,8 @@ public class FullTextLucene extends FullText { ...@@ -541,9 +531,8 @@ public class FullTextLucene extends FullText {
if (rs.next()) { if (rs.next()) {
String cols = rs.getString(1); String cols = rs.getString(1);
if (cols != null) { if (cols != null) {
for (String s : StringUtils.arraySplit(cols, ',', true)) { Collections.addAll(indexList,
indexList.add(s); StringUtils.arraySplit(cols, ',', true));
}
} }
} }
if (indexList.size() == 0) { if (indexList.size() == 0) {
...@@ -583,10 +572,7 @@ public class FullTextLucene extends FullText { ...@@ -583,10 +572,7 @@ public class FullTextLucene extends FullText {
*/ */
@Override @Override
public void close() throws SQLException { public void close() throws SQLException {
if (indexAccess != null) { removeIndexAccess(indexPath);
removeIndexAccess(indexAccess, indexPath);
indexAccess = null;
}
} }
/** /**
...@@ -600,14 +586,9 @@ public class FullTextLucene extends FullText { ...@@ -600,14 +586,9 @@ public class FullTextLucene extends FullText {
/** /**
* Commit all changes to the Lucene index. * Commit all changes to the Lucene index.
*/ */
void commitIndex() throws SQLException { private void commitIndex() throws SQLException {
try { try {
indexAccess.writer.commit(); indexAccess.commit();
// recreate Searcher with the IndexWriter's reader.
indexAccess.searcher.close();
indexAccess.reader.close();
indexAccess.reader = IndexReader.open(indexAccess.writer, true);
indexAccess.searcher = new IndexSearcher(indexAccess.reader);
} catch (IOException e) { } catch (IOException e) {
throw convertException(e); throw convertException(e);
} }
...@@ -699,22 +680,80 @@ public class FullTextLucene extends FullText { ...@@ -699,22 +680,80 @@ public class FullTextLucene extends FullText {
/** /**
* A wrapper for the Lucene writer and searcher. * A wrapper for the Lucene writer and searcher.
*/ */
static class IndexAccess { static final class IndexAccess {
/** /**
* The index writer. * The index writer.
*/ */
IndexWriter writer; final IndexWriter writer;
/** /**
* The index reader. * Map of usage counters for outstanding searchers.
*/ */
IndexReader reader; private final Map<IndexSearcher,Integer> counters = New.hashMap();
/**
* Usage counter for current searcher.
*/
private int counter;
/** /**
* The index searcher. * The index searcher.
*/ */
IndexSearcher searcher; private IndexSearcher searcher;
}
private IndexAccess(IndexWriter writer) throws IOException {
this.writer = writer;
IndexReader reader = IndexReader.open(writer, true);
searcher = new IndexSearcher(reader);
}
private synchronized IndexSearcher getSearcher() {
++counter;
return searcher;
}
private synchronized void returnSearcher(IndexSearcher searcher) {
if (this.searcher == searcher) {
--counter;
assert counter >= 0;
} else {
Integer cnt = counters.remove(searcher);
assert cnt != null;
if(--cnt == 0) {
closeSearcher(searcher);
} else {
counters.put(searcher, cnt);
}
}
}
public synchronized void commit() throws IOException {
writer.commit();
if (counter != 0) {
counters.put(searcher, counter);
counter = 0;
} else {
closeSearcher(searcher);
}
// recreate Searcher with the IndexWriter's reader.
searcher = new IndexSearcher(IndexReader.open(writer, true));
}
public synchronized void close() throws IOException {
for (IndexSearcher searcher : counters.keySet()) {
closeSearcher(searcher);
}
counters.clear();
closeSearcher(searcher);
searcher = null;
writer.close();
}
private static void closeSearcher(IndexSearcher searcher) {
IndexReader indexReader = searcher.getIndexReader();
try { searcher.close(); } catch(IOException ignore) {/**/}
try { indexReader.close(); } catch(IOException ignore) {/**/}
}
}
} }
...@@ -10,20 +10,22 @@ import java.sql.PreparedStatement; ...@@ -10,20 +10,22 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.HashMap; import java.util.Collections;
import java.util.HashSet; import java.util.Map;
import java.util.Set;
import org.h2.util.New; import org.h2.util.New;
import org.h2.util.SoftHashMap; import org.h2.util.SoftHashMap;
/** /**
* The global settings of a full text search. * The global settings of a full text search.
*/ */
class FullTextSettings { final class FullTextSettings {
/** /**
* The settings of open indexes. * The settings of open indexes.
*/ */
private static final HashMap<String, FullTextSettings> SETTINGS = New.hashMap(); private static final Map<String, FullTextSettings> SETTINGS = New.hashMap();
/** /**
* Whether this instance has been initialized. * Whether this instance has been initialized.
...@@ -33,17 +35,17 @@ class FullTextSettings { ...@@ -33,17 +35,17 @@ class FullTextSettings {
/** /**
* The set of words not to index (stop words). * The set of words not to index (stop words).
*/ */
private final HashSet<String> ignoreList = New.hashSet(); private final Set<String> ignoreList = New.hashSet();
/** /**
* The set of words / terms. * The set of words / terms.
*/ */
private final HashMap<String, Integer> words = New.hashMap(); private final Map<String, Integer> words = New.hashMap();
/** /**
* The set of indexes in this database. * The set of indexes in this database.
*/ */
private final HashMap<Integer, IndexInfo> indexes = New.hashMap(); private final Map<Integer, IndexInfo> indexes = Collections.synchronizedMap(New.<Integer, IndexInfo>hashMap());
/** /**
* The prepared statement cache. * The prepared statement cache.
...@@ -60,26 +62,63 @@ class FullTextSettings { ...@@ -60,26 +62,63 @@ class FullTextSettings {
/** /**
* Create a new instance. * Create a new instance.
*/ */
protected FullTextSettings() { private FullTextSettings() {
// don't allow construction // don't allow construction
} }
/** /**
* Get the ignore list. * Clear set of ignored words
*
* @return the ignore list
*/ */
protected HashSet<String> getIgnoreList() { public void clearInored() {
return ignoreList; synchronized (ignoreList) {
ignoreList.clear();
}
} }
/** /**
* Get the word list. * Amend set of ignored words
* * @param words to add
* @return the word list */
public void addIgnored(Iterable<String> words) {
synchronized (ignoreList) {
for (String word : words) {
word = normalizeWord(word);
ignoreList.add(word);
}
}
}
/**
* Clear set of searchable words
*/
public void clearWordList() {
synchronized (words) {
words.clear();
}
}
/**
* Get id for a searchable word
* @param word to find id for
* @return Integer id or null if word is not found
*/
public Integer getWordId(String word) {
synchronized (words) {
return words.get(word);
}
}
/**
* Register searchable word
* @param word to register
* @param id to register with
*/ */
protected HashMap<String, Integer> getWordList() { public void addWord(String word, Integer id) {
return words; synchronized (words) {
if(!words.containsKey(word)) {
words.put(word, id);
}
}
} }
/** /**
...@@ -109,10 +148,11 @@ class FullTextSettings { ...@@ -109,10 +148,11 @@ class FullTextSettings {
* @return the uppercase version of the word or null * @return the uppercase version of the word or null
*/ */
protected String convertWord(String word) { protected String convertWord(String word) {
// TODO this is locale specific, document word = normalizeWord(word);
word = word.toUpperCase(); synchronized (ignoreList) {
if (ignoreList.contains(word)) { if (ignoreList.contains(word)) {
return null; return null;
}
} }
return word; return word;
} }
...@@ -126,10 +166,13 @@ class FullTextSettings { ...@@ -126,10 +166,13 @@ class FullTextSettings {
protected static FullTextSettings getInstance(Connection conn) protected static FullTextSettings getInstance(Connection conn)
throws SQLException { throws SQLException {
String path = getIndexPath(conn); String path = getIndexPath(conn);
FullTextSettings setting = SETTINGS.get(path); FullTextSettings setting;
if (setting == null) { synchronized (SETTINGS) {
setting = new FullTextSettings(); setting = SETTINGS.get(path);
SETTINGS.put(path, setting); if (setting == null) {
setting = new FullTextSettings();
SETTINGS.put(path, setting);
}
} }
return setting; return setting;
} }
...@@ -140,7 +183,7 @@ class FullTextSettings { ...@@ -140,7 +183,7 @@ class FullTextSettings {
* @param conn the connection * @param conn the connection
* @return the file system path * @return the file system path
*/ */
protected static String getIndexPath(Connection conn) throws SQLException { private static String getIndexPath(Connection conn) throws SQLException {
Statement stat = conn.createStatement(); Statement stat = conn.createStatement();
ResultSet rs = stat.executeQuery( ResultSet rs = stat.executeQuery(
"CALL IFNULL(DATABASE_PATH(), 'MEM:' || DATABASE())"); "CALL IFNULL(DATABASE_PATH(), 'MEM:' || DATABASE())");
...@@ -218,7 +261,9 @@ class FullTextSettings { ...@@ -218,7 +261,9 @@ class FullTextSettings {
* Close all fulltext settings, freeing up memory. * Close all fulltext settings, freeing up memory.
*/ */
protected static void closeAll() { protected static void closeAll() {
SETTINGS.clear(); synchronized (SETTINGS) {
SETTINGS.clear();
}
} }
protected void setWhitespaceChars(String whitespaceChars) { protected void setWhitespaceChars(String whitespaceChars) {
...@@ -229,4 +274,8 @@ class FullTextSettings { ...@@ -229,4 +274,8 @@ class FullTextSettings {
return whitespaceChars; return whitespaceChars;
} }
private String normalizeWord(String word) {
// TODO this is locale specific, document
return word.toUpperCase();
}
} }
...@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; ...@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit;
import org.h2.fulltext.FullText; import org.h2.fulltext.FullText;
import org.h2.store.fs.FileUtils; import org.h2.store.fs.FileUtils;
import org.h2.test.TestAll;
import org.h2.test.TestBase; import org.h2.test.TestBase;
import org.h2.util.IOUtils; import org.h2.util.IOUtils;
import org.h2.util.Task; import org.h2.util.Task;
...@@ -50,10 +51,6 @@ public class TestFullText extends TestBase { ...@@ -50,10 +51,6 @@ public class TestFullText extends TestBase {
@Override @Override
public void test() throws Exception { public void test() throws Exception {
if (config.multiThreaded) {
// It is even mentioned in the docs that this is not supported
return;
}
testUuidPrimaryKey(false); testUuidPrimaryKey(false);
testAutoAnalyze(); testAutoAnalyze();
testNativeFeatures(); testNativeFeatures();
...@@ -71,7 +68,9 @@ public class TestFullText extends TestBase { ...@@ -71,7 +68,9 @@ public class TestFullText extends TestBase {
testCreateDropLucene(); testCreateDropLucene();
testUuidPrimaryKey(true); testUuidPrimaryKey(true);
testMultiThreaded(true); testMultiThreaded(true);
testMultiThreaded(false); if(config.mvStore || !config.multiThreaded) {
testMultiThreaded(false);
}
testTransaction(true); testTransaction(true);
test(true, "VARCHAR"); test(true, "VARCHAR");
test(true, "CLOB"); test(true, "CLOB");
...@@ -256,7 +255,7 @@ public class TestFullText extends TestBase { ...@@ -256,7 +255,7 @@ public class TestFullText extends TestBase {
int len = 2; int len = 2;
Task[] task = new Task[len]; Task[] task = new Task[len];
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
final Connection conn = getConnection("fullText", connList); final Connection conn = getConnection("fullText;LOCK_TIMEOUT=60000", connList);
Statement stat = conn.createStatement(); Statement stat = conn.createStatement();
initFullText(stat, lucene); initFullText(stat, lucene);
initFullText(stat, lucene); initFullText(stat, lucene);
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论