Skip to content

H
h2database
提交 e1a91d29 authored 作者: andrei's avatar andrei
浏览文件
操作

full_text_mt

上级 a24af894
隐藏空白字符变更
内嵌 并排
正在显示 包含 354 行增加 和 248 行删除
h2/src/main/org/h2/fulltext/FullText.java
浏览文件 @ e1a91d29
...@@ -18,11 +18,13 @@ import java.sql.Statement; ...@@ -18,11 +18,13 @@ import java.sql.Statement;
import java.sql.Types; import java.sql.Types;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer; import java.util.StringTokenizer;
import java.util.UUID; import java.util.UUID;
import org.h2.api.Trigger; import org.h2.api.Trigger;
import org.h2.command.Parser; import org.h2.command.Parser;
import org.h2.engine.Session; import org.h2.engine.Session;
...@@ -151,7 +153,7 @@ public class FullText { ...@@ -151,7 +153,7 @@ public class FullText {
} }
} }
rs = stat.executeQuery("SELECT * FROM " + SCHEMA + ".WORDS"); rs = stat.executeQuery("SELECT * FROM " + SCHEMA + ".WORDS");
HashMap<String, Integer> map = setting.getWordList(); Map<String, Integer> map = setting.getWordList();
while (rs.next()) { while (rs.next()) {
String word = rs.getString("NAME"); String word = rs.getString("NAME");
int id = rs.getInt("ID"); int id = rs.getInt("ID");
...@@ -243,9 +245,9 @@ public class FullText { ...@@ -243,9 +245,9 @@ public class FullText {
break; break;
} }
} }
prep = conn.prepareStatement("DELETE FROM " + SCHEMA + ".MAP M " + prep = conn.prepareStatement("DELETE FROM " + SCHEMA + ".MAP " +
"WHERE NOT EXISTS (SELECT * FROM " + SCHEMA + "WHERE NOT EXISTS (SELECT * FROM " + SCHEMA +
".ROWS R WHERE R.ID=M.ROWID) AND ROWID<10000"); ".ROWS R WHERE R.ID=ROWID) AND ROWID<10000");
while (true) { while (true) {
int deleted = prep.executeUpdate(); int deleted = prep.executeUpdate();
if (deleted == 0) { if (deleted == 0) {
...@@ -605,10 +607,10 @@ public class FullText { ...@@ -605,10 +607,10 @@ public class FullText {
if (!setting.isInitialized()) { if (!setting.isInitialized()) {
init(conn); init(conn);
} }
HashSet<String> words = New.hashSet(); Set<String> words = New.hashSet();
addWords(setting, words, text); addWords(setting, words, text);
HashSet<Integer> rIds = null, lastRowIds = null; Set<Integer> rIds = null, lastRowIds;
HashMap<String, Integer> allWords = setting.getWordList(); Map<String, Integer> allWords = setting.getWordList();
PreparedStatement prepSelectMapByWordId = setting.prepare(conn, PreparedStatement prepSelectMapByWordId = setting.prepare(conn,
SELECT_MAP_BY_WORD_ID); SELECT_MAP_BY_WORD_ID);
...@@ -698,7 +700,7 @@ public class FullText { ...@@ -698,7 +700,7 @@ public class FullText {
* @param reader the reader * @param reader the reader
*/ */
protected static void addWords(FullTextSettings setting, protected static void addWords(FullTextSettings setting,
HashSet<String> set, Reader reader) { Set<String> set, Reader reader) {
StreamTokenizer tokenizer = new StreamTokenizer(reader); StreamTokenizer tokenizer = new StreamTokenizer(reader);
tokenizer.resetSyntax(); tokenizer.resetSyntax();
tokenizer.wordChars(' ' + 1, 255); tokenizer.wordChars(' ' + 1, 255);
...@@ -732,7 +734,7 @@ public class FullText { ...@@ -732,7 +734,7 @@ public class FullText {
* @param text the text * @param text the text
*/ */
protected static void addWords(FullTextSettings setting, protected static void addWords(FullTextSettings setting,
HashSet<String> set, String text) { Set<String> set, String text) {
String whitespaceChars = setting.getWhitespaceChars(); String whitespaceChars = setting.getWhitespaceChars();
StringTokenizer tokenizer = new StringTokenizer(text, whitespaceChars); StringTokenizer tokenizer = new StringTokenizer(text, whitespaceChars);
while (tokenizer.hasMoreTokens()) { while (tokenizer.hasMoreTokens()) {
...@@ -751,30 +753,38 @@ public class FullText { ...@@ -751,30 +753,38 @@ public class FullText {
* @param schema the schema name * @param schema the schema name
* @param table the table name * @param table the table name
*/ */
protected static void createTrigger(Connection conn, String schema, private static void createTrigger(Connection conn, String schema,
String table) throws SQLException { String table) throws SQLException {
createOrDropTrigger(conn, schema, table, true); createOrDropTrigger(conn, schema, table, true);
} }
private static void createOrDropTrigger(Connection conn, private static void createOrDropTrigger(Connection conn,
String schema, String table, boolean create) throws SQLException { String schema, String table, boolean create) throws SQLException {
Statement stat = conn.createStatement(); try (Statement stat = conn.createStatement()) {
String trigger = StringUtils.quoteIdentifier(schema) + "." String trigger = StringUtils.quoteIdentifier(schema) + "."
+ StringUtils.quoteIdentifier(TRIGGER_PREFIX + table); + StringUtils.quoteIdentifier(TRIGGER_PREFIX + table);
stat.execute("DROP TRIGGER IF EXISTS " + trigger); stat.execute("DROP TRIGGER IF EXISTS " + trigger);
if (create) { if (create) {
StringBuilder buff = new StringBuilder("CREATE TRIGGER IF NOT EXISTS "); ResultSet rs = stat.executeQuery("SELECT value FROM information_schema.settings WHERE name = 'MV_STORE'");
// needs to be called on rollback as well, because we use the init boolean multiThread = FullTextTrigger.isMultiThread(conn);
// connection do to changes in the index (not the user connection) StringBuilder buff = new StringBuilder("CREATE TRIGGER IF NOT EXISTS ");
buff.append(trigger). // unless multithread, trigger needs to be called on rollback as well,
append(" AFTER INSERT, UPDATE, DELETE, ROLLBACK ON "). // because we use the init connection do to changes in the index
append(StringUtils.quoteIdentifier(schema)). // (not the user connection)
append('.'). buff.append(trigger).
append(StringUtils.quoteIdentifier(table)). append(" AFTER INSERT, UPDATE, DELETE");
append(" FOR EACH ROW CALL \""). if(!multiThread) {
append(FullText.FullTextTrigger.class.getName()). buff.append(", ROLLBACK");
append('\"'); }
stat.execute(buff.toString()); buff.append(" ON ").
append(StringUtils.quoteIdentifier(schema)).
append('.').
append(StringUtils.quoteIdentifier(table)).
append(" FOR EACH ROW CALL \"").
append(FullText.FullTextTrigger.class.getName()).
append('\"');
stat.execute(buff.toString());
}
} }
} }
...@@ -785,8 +795,8 @@ public class FullText { ...@@ -785,8 +795,8 @@ public class FullText {
* @param schema the schema name * @param schema the schema name
* @param table the table name * @param table the table name
*/ */
protected static void indexExistingRows(Connection conn, String schema, private static void indexExistingRows(Connection conn, String schema,
String table) throws SQLException { String table) throws SQLException {
FullText.FullTextTrigger existing = new FullText.FullTextTrigger(); FullText.FullTextTrigger existing = new FullText.FullTextTrigger();
existing.init(conn, schema, null, table, false, Trigger.INSERT); existing.init(conn, schema, null, table, false, Trigger.INSERT);
String sql = "SELECT * FROM " + StringUtils.quoteIdentifier(schema) + String sql = "SELECT * FROM " + StringUtils.quoteIdentifier(schema) +
...@@ -823,7 +833,7 @@ public class FullText { ...@@ -823,7 +833,7 @@ public class FullText {
private static void setIgnoreList(FullTextSettings setting, private static void setIgnoreList(FullTextSettings setting,
String commaSeparatedList) { String commaSeparatedList) {
String[] list = StringUtils.arraySplit(commaSeparatedList, ',', true); String[] list = StringUtils.arraySplit(commaSeparatedList, ',', true);
HashSet<String> set = setting.getIgnoreList(); Set<String> set = setting.getIgnoreList();
for (String word : list) { for (String word : list) {
String converted = setting.convertWord(word); String converted = setting.convertWord(word);
if (converted != null) { if (converted != null) {
...@@ -860,14 +870,28 @@ public class FullText { ...@@ -860,14 +870,28 @@ public class FullText {
/** /**
* Trigger updates the index when a inserting, updating, or deleting a row. * Trigger updates the index when a inserting, updating, or deleting a row.
*/ */
public static class FullTextTrigger implements Trigger { public static final class FullTextTrigger implements Trigger {
private FullTextSettings setting;
private IndexInfo index;
private int[] columnTypes;
private final PreparedStatement[] prepStatements = new PreparedStatement[SQL.length];
private boolean useOwnConnection;
protected FullTextSettings setting; private static final int INSERT_WORD = 0;
protected IndexInfo index; private static final int INSERT_ROW = 1;
protected int[] columnTypes; private static final int INSERT_MAP = 2;
protected PreparedStatement prepInsertWord, prepInsertRow, prepInsertMap; private static final int DELETE_ROW = 3;
protected PreparedStatement prepDeleteRow, prepDeleteMap; private static final int DELETE_MAP = 4;
protected PreparedStatement prepSelectRow; private static final int SELECT_ROW = 5;
private static final String SQL[] = {
"INSERT INTO " + SCHEMA + ".WORDS(NAME) VALUES(?)",
"INSERT INTO " + SCHEMA + ".ROWS(HASH, INDEXID, KEY) VALUES(?, ?, ?)",
"INSERT INTO " + SCHEMA + ".MAP(ROWID, WORDID) VALUES(?, ?)",
"DELETE FROM " + SCHEMA + ".ROWS WHERE HASH=? AND INDEXID=? AND KEY=?",
"DELETE FROM " + SCHEMA + ".MAP WHERE ROWID=? AND WORDID=?",
"SELECT ID FROM " + SCHEMA + ".ROWS WHERE HASH=? AND INDEXID=? AND KEY=?"
};
/** /**
* INTERNAL * INTERNAL
...@@ -923,9 +947,7 @@ public class FullText { ...@@ -923,9 +947,7 @@ public class FullText {
index.id = rs.getInt(1); index.id = rs.getInt(1);
String columns = rs.getString(2); String columns = rs.getString(2);
if (columns != null) { if (columns != null) {
for (String s : StringUtils.arraySplit(columns, ',', true)) { Collections.addAll(indexList, StringUtils.arraySplit(columns, ',', true));
indexList.add(s);
}
} }
} }
if (indexList.size() == 0) { if (indexList.size() == 0) {
...@@ -936,18 +958,23 @@ public class FullText { ...@@ -936,18 +958,23 @@ public class FullText {
index.indexColumns = new int[indexList.size()]; index.indexColumns = new int[indexList.size()];
setColumns(index.indexColumns, indexList, columnList); setColumns(index.indexColumns, indexList, columnList);
setting.addIndexInfo(index); setting.addIndexInfo(index);
prepInsertWord = conn.prepareStatement(
"INSERT INTO " + SCHEMA + ".WORDS(NAME) VALUES(?)"); useOwnConnection = isMultiThread(conn);
prepInsertRow = conn.prepareStatement( if(!useOwnConnection) {
"INSERT INTO " + SCHEMA + ".ROWS(HASH, INDEXID, KEY) VALUES(?, ?, ?)"); for (int i = 0; i < SQL.length; i++) {
prepInsertMap = conn.prepareStatement( prepStatements[i] = conn.prepareStatement(SQL[i]);
"INSERT INTO " + SCHEMA + ".MAP(ROWID, WORDID) VALUES(?, ?)"); }
prepDeleteRow = conn.prepareStatement( }
"DELETE FROM " + SCHEMA + ".ROWS WHERE HASH=? AND INDEXID=? AND KEY=?"); }
prepDeleteMap = conn.prepareStatement(
"DELETE FROM " + SCHEMA + ".MAP WHERE ROWID=? AND WORDID=?"); private static boolean isMultiThread(Connection conn)
prepSelectRow = conn.prepareStatement( throws SQLException {
"SELECT ID FROM " + SCHEMA + ".ROWS WHERE HASH=? AND INDEXID=? AND KEY=?"); try (Statement stat = conn.createStatement()) {
ResultSet rs = stat.executeQuery(
"SELECT value FROM information_schema.settings" +
" WHERE name = 'MULTI_THREADED'");
return rs.next() && !"0".equals(rs.getString(1));
}
} }
/** /**
...@@ -960,16 +987,16 @@ public class FullText { ...@@ -960,16 +987,16 @@ public class FullText {
if (newRow != null) { if (newRow != null) {
// update // update
if (hasChanged(oldRow, newRow, index.indexColumns)) { if (hasChanged(oldRow, newRow, index.indexColumns)) {
delete(oldRow); delete(conn, oldRow);
insert(newRow); insert(conn, newRow);
} }
} else { } else {
// delete // delete
delete(oldRow); delete(conn, oldRow);
} }
} else if (newRow != null) { } else if (newRow != null) {
// insert // insert
insert(newRow); insert(conn, newRow);
} }
} }
...@@ -992,54 +1019,82 @@ public class FullText { ...@@ -992,54 +1019,82 @@ public class FullText {
/** /**
* Add a row to the index. * Add a row to the index.
* *
* @param conn to use
* @param row the row * @param row the row
*/ */
protected void insert(Object[] row) throws SQLException { protected void insert(Connection conn, Object[] row) throws SQLException {
String key = getKey(row); PreparedStatement prepInsertRow = null;
int hash = key.hashCode(); PreparedStatement prepInsertMap = null;
prepInsertRow.setInt(1, hash); try {
prepInsertRow.setInt(2, index.id); String key = getKey(row);
prepInsertRow.setString(3, key); int hash = key.hashCode();
prepInsertRow.execute(); prepInsertRow = getStatement(conn, INSERT_ROW);
ResultSet rs = prepInsertRow.getGeneratedKeys(); prepInsertRow.setInt(1, hash);
rs.next(); prepInsertRow.setInt(2, index.id);
int rowId = rs.getInt(1); prepInsertRow.setString(3, key);
prepInsertMap.setInt(1, rowId); prepInsertRow.execute();
int[] wordIds = getWordIds(row); ResultSet rs = prepInsertRow.getGeneratedKeys();
for (int id : wordIds) { rs.next();
prepInsertMap.setInt(2, id); int rowId = rs.getInt(1);
prepInsertMap.execute();
prepInsertMap = getStatement(conn, INSERT_MAP);
prepInsertMap.setInt(1, rowId);
int[] wordIds = getWordIds(conn, row);
for (int id : wordIds) {
prepInsertMap.setInt(2, id);
prepInsertMap.execute();
}
} finally {
if (useOwnConnection) {
IOUtils.closeSilently(prepInsertRow);
IOUtils.closeSilently(prepInsertMap);
}
} }
} }
/** /**
* Delete a row from the index. * Delete a row from the index.
* *
* @param conn to use
* @param row the row * @param row the row
*/ */
protected void delete(Object[] row) throws SQLException { protected void delete(Connection conn, Object[] row) throws SQLException {
String key = getKey(row); PreparedStatement prepSelectRow = null;
int hash = key.hashCode(); PreparedStatement prepDeleteMap = null;
prepSelectRow.setInt(1, hash); PreparedStatement prepDeleteRow = null;
prepSelectRow.setInt(2, index.id); try {
prepSelectRow.setString(3, key); String key = getKey(row);
ResultSet rs = prepSelectRow.executeQuery(); int hash = key.hashCode();
if (rs.next()) { prepSelectRow = getStatement(conn, SELECT_ROW);
int rowId = rs.getInt(1); prepSelectRow.setInt(1, hash);
prepDeleteMap.setInt(1, rowId); prepSelectRow.setInt(2, index.id);
int[] wordIds = getWordIds(row); prepSelectRow.setString(3, key);
for (int id : wordIds) { ResultSet rs = prepSelectRow.executeQuery();
prepDeleteMap.setInt(2, id); prepDeleteMap = getStatement(conn, DELETE_MAP);
prepDeleteMap.executeUpdate(); prepDeleteRow = getStatement(conn, DELETE_ROW);
if (rs.next()) {
int rowId = rs.getInt(1);
prepDeleteMap.setInt(1, rowId);
int[] wordIds = getWordIds(conn, row);
for (int id : wordIds) {
prepDeleteMap.setInt(2, id);
prepDeleteMap.executeUpdate();
}
prepDeleteRow.setInt(1, hash);
prepDeleteRow.setInt(2, index.id);
prepDeleteRow.setString(3, key);
prepDeleteRow.executeUpdate();
}
} finally {
if (useOwnConnection) {
IOUtils.closeSilently(prepSelectRow);
IOUtils.closeSilently(prepDeleteMap);
IOUtils.closeSilently(prepDeleteRow);
} }
prepDeleteRow.setInt(1, hash);
prepDeleteRow.setInt(2, index.id);
prepDeleteRow.setString(3, key);
prepDeleteRow.executeUpdate();
} }
} }
private int[] getWordIds(Object[] row) throws SQLException { private int[] getWordIds(Connection conn, Object[] row) throws SQLException {
HashSet<String> words = New.hashSet(); HashSet<String> words = New.hashSet();
for (int idx : index.indexColumns) { for (int idx : index.indexColumns) {
int type = columnTypes[idx]; int type = columnTypes[idx];
...@@ -1057,27 +1112,36 @@ public class FullText { ...@@ -1057,27 +1112,36 @@ public class FullText {
addWords(setting, words, string); addWords(setting, words, string);
} }
} }
HashMap<String, Integer> allWords = setting.getWordList(); PreparedStatement prepInsertWord = null;
int[] wordIds = new int[words.size()]; try {
Iterator<String> it = words.iterator(); prepInsertWord = getStatement(conn, INSERT_WORD);
for (int i = 0; it.hasNext(); i++) { Map<String, Integer> allWords = setting.getWordList();
String word = it.next(); int[] wordIds = new int[words.size()];
Integer wId = allWords.get(word); synchronized (allWords) {
int wordId; int i = 0;
if (wId == null) { for (String word : words) {
prepInsertWord.setString(1, word); Integer wId = allWords.get(word);
prepInsertWord.execute(); int wordId;
ResultSet rs = prepInsertWord.getGeneratedKeys(); if (wId == null) {
rs.next(); prepInsertWord.setString(1, word);
wordId = rs.getInt(1); prepInsertWord.execute();
allWords.put(word, wordId); ResultSet rs = prepInsertWord.getGeneratedKeys();
} else { rs.next();
wordId = wId.intValue(); wordId = rs.getInt(1);
allWords.put(word, wordId);
} else {
wordId = wId;
}
wordIds[i++] = wordId;
}
}
Arrays.sort(wordIds);
return wordIds;
} finally {
if (useOwnConnection) {
IOUtils.closeSilently(prepInsertWord);
} }
wordIds[i] = wordId;
} }
Arrays.sort(wordIds);
return wordIds;
} }
private String getKey(Object[] row) throws SQLException { private String getKey(Object[] row) throws SQLException {
...@@ -1095,13 +1159,17 @@ public class FullText { ...@@ -1095,13 +1159,17 @@ public class FullText {
return buff.toString(); return buff.toString();
} }
private PreparedStatement getStatement(Connection conn, int indx) throws SQLException {
return useOwnConnection ? conn.prepareStatement(SQL[indx]) : prepStatements[indx];
}
} }
/** /**
* INTERNAL * INTERNAL
* Close all fulltext settings, freeing up memory. * Close all fulltext settings, freeing up memory.
*/ */
public static void closeAll() { public static synchronized void closeAll() {
FullTextSettings.closeAll(); FullTextSettings.closeAll();
} }
...@@ -1116,5 +1184,4 @@ public class FullText { ...@@ -1116,5 +1184,4 @@ public class FullText {
throws SQLException { throws SQLException {
throw new SQLException(message, "FULLTEXT"); throw new SQLException(message, "FULLTEXT");
} }
} }
h2/src/main/org/h2/fulltext/FullTextLucene.java
浏览文件 @ e1a91d29
...@@ -13,6 +13,7 @@ import java.sql.ResultSet; ...@@ -13,6 +13,7 @@ import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer;
...@@ -37,6 +38,8 @@ import org.h2.util.StatementBuilder; ...@@ -37,6 +38,8 @@ import org.h2.util.StatementBuilder;
import org.h2.util.StringUtils; import org.h2.util.StringUtils;
import org.h2.util.Utils; import org.h2.util.Utils;
import java.io.File; import java.io.File;
import java.util.Map;
import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FSDirectory;
...@@ -94,27 +97,23 @@ public class FullTextLucene extends FullText { ...@@ -94,27 +97,23 @@ public class FullTextLucene extends FullText {
* @param conn the connection * @param conn the connection
*/ */
public static void init(Connection conn) throws SQLException { public static void init(Connection conn) throws SQLException {
Statement stat = conn.createStatement(); try (Statement stat = conn.createStatement()) {
stat.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); stat.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA);
stat.execute("CREATE TABLE IF NOT EXISTS " + SCHEMA + stat.execute("CREATE TABLE IF NOT EXISTS " + SCHEMA +
".INDEXES(SCHEMA VARCHAR, TABLE VARCHAR, " + ".INDEXES(SCHEMA VARCHAR, TABLE VARCHAR, " +
"COLUMNS VARCHAR, PRIMARY KEY(SCHEMA, TABLE))"); "COLUMNS VARCHAR, PRIMARY KEY(SCHEMA, TABLE))");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_CREATE_INDEX FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_CREATE_INDEX FOR \"" +
FullTextLucene.class.getName() + ".createIndex\""); FullTextLucene.class.getName() + ".createIndex\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_DROP_INDEX FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_DROP_INDEX FOR \"" +
FullTextLucene.class.getName() + ".dropIndex\""); FullTextLucene.class.getName() + ".dropIndex\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_SEARCH FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_SEARCH FOR \"" +
FullTextLucene.class.getName() + ".search\""); FullTextLucene.class.getName() + ".search\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_SEARCH_DATA FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_SEARCH_DATA FOR \"" +
FullTextLucene.class.getName() + ".searchData\""); FullTextLucene.class.getName() + ".searchData\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_REINDEX FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_REINDEX FOR \"" +
FullTextLucene.class.getName() + ".reindex\""); FullTextLucene.class.getName() + ".reindex\"");
stat.execute("CREATE ALIAS IF NOT EXISTS FTL_DROP_ALL FOR \"" + stat.execute("CREATE ALIAS IF NOT EXISTS FTL_DROP_ALL FOR \"" +
FullTextLucene.class.getName() + ".dropAll\""); FullTextLucene.class.getName() + ".dropAll\"");
try {
getIndexAccess(conn);
} catch (SQLException e) {
throw convertException(e);
} }
} }
...@@ -157,11 +156,9 @@ public class FullTextLucene extends FullText { ...@@ -157,11 +156,9 @@ public class FullTextLucene extends FullText {
prep.setString(1, schema); prep.setString(1, schema);
prep.setString(2, table); prep.setString(2, table);
int rowCount = prep.executeUpdate(); int rowCount = prep.executeUpdate();
if (rowCount == 0) { if (rowCount != 0) {
return; reindex(conn);
} }
reindex(conn);
} }
/** /**
...@@ -248,10 +245,7 @@ public class FullTextLucene extends FullText { ...@@ -248,10 +245,7 @@ public class FullTextLucene extends FullText {
* @return the converted SQL exception * @return the converted SQL exception
*/ */
protected static SQLException convertException(Exception e) { protected static SQLException convertException(Exception e) {
SQLException e2 = new SQLException( return new SQLException("Error while indexing document", "FULLTEXT", e);
"Error while indexing document", "FULLTEXT");
e2.initCause(e);
return e2;
} }
/** /**
...@@ -261,8 +255,8 @@ public class FullTextLucene extends FullText { ...@@ -261,8 +255,8 @@ public class FullTextLucene extends FullText {
* @param schema the schema name * @param schema the schema name
* @param table the table name * @param table the table name
*/ */
protected static void createTrigger(Connection conn, String schema, private static void createTrigger(Connection conn, String schema,
String table) throws SQLException { String table) throws SQLException {
createOrDropTrigger(conn, schema, table, true); createOrDropTrigger(conn, schema, table, true);
} }
...@@ -309,11 +303,7 @@ public class FullTextLucene extends FullText { ...@@ -309,11 +303,7 @@ public class FullTextLucene extends FullText {
conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
IndexWriter writer = new IndexWriter(indexDir, conf); IndexWriter writer = new IndexWriter(indexDir, conf);
//see http://wiki.apache.org/lucene-java/NearRealtimeSearch //see http://wiki.apache.org/lucene-java/NearRealtimeSearch
IndexReader reader = IndexReader.open(writer, true); access = new IndexAccess(writer);
access = new IndexAccess();
access.writer = writer;
access.reader = reader;
access.searcher = new IndexSearcher(reader);
} catch (IOException e) { } catch (IOException e) {
throw convertException(e); throw convertException(e);
} }
...@@ -353,8 +343,8 @@ public class FullTextLucene extends FullText { ...@@ -353,8 +343,8 @@ public class FullTextLucene extends FullText {
* @param schema the schema name * @param schema the schema name
* @param table the table name * @param table the table name
*/ */
protected static void indexExistingRows(Connection conn, String schema, private static void indexExistingRows(Connection conn, String schema,
String table) throws SQLException { String table) throws SQLException {
FullTextLucene.FullTextTrigger existing = new FullTextLucene.FullTextTrigger(); FullTextLucene.FullTextTrigger existing = new FullTextLucene.FullTextTrigger();
existing.init(conn, schema, null, table, false, Trigger.INSERT); existing.init(conn, schema, null, table, false, Trigger.INSERT);
String sql = "SELECT * FROM " + StringUtils.quoteIdentifier(schema) + String sql = "SELECT * FROM " + StringUtils.quoteIdentifier(schema) +
...@@ -373,10 +363,7 @@ public class FullTextLucene extends FullText { ...@@ -373,10 +363,7 @@ public class FullTextLucene extends FullText {
private static void removeIndexFiles(Connection conn) throws SQLException { private static void removeIndexFiles(Connection conn) throws SQLException {
String path = getIndexPath(conn); String path = getIndexPath(conn);
IndexAccess access = INDEX_ACCESS.get(path); removeIndexAccess(path);
if (access != null) {
removeIndexAccess(access, path);
}
if (!path.startsWith(IN_MEMORY_PREFIX)) { if (!path.startsWith(IN_MEMORY_PREFIX)) {
FileUtils.deleteRecursive(path, false); FileUtils.deleteRecursive(path, false);
} }
...@@ -386,17 +373,16 @@ public class FullTextLucene extends FullText { ...@@ -386,17 +373,16 @@ public class FullTextLucene extends FullText {
* Close the index writer and searcher and remove them from the index access * Close the index writer and searcher and remove them from the index access
* set. * set.
* *
* @param access the index writer/searcher wrapper
* @param indexPath the index path * @param indexPath the index path
*/ */
protected static void removeIndexAccess(IndexAccess access, String indexPath) protected static void removeIndexAccess(String indexPath)
throws SQLException { throws SQLException {
synchronized (INDEX_ACCESS) { synchronized (INDEX_ACCESS) {
try { try {
INDEX_ACCESS.remove(indexPath); IndexAccess access = INDEX_ACCESS.remove(indexPath);
access.searcher.close(); if(access != null) {
access.reader.close(); access.close();
access.writer.close(); }
} catch (Exception e) { } catch (Exception e) {
throw convertException(e); throw convertException(e);
} }
...@@ -426,49 +412,53 @@ public class FullTextLucene extends FullText { ...@@ -426,49 +412,53 @@ public class FullTextLucene extends FullText {
try { try {
IndexAccess access = getIndexAccess(conn); IndexAccess access = getIndexAccess(conn);
// take a reference as the searcher may change // take a reference as the searcher may change
IndexSearcher searcher = access.searcher; IndexSearcher searcher = access.getSearcher();
// reuse the same analyzer; it's thread-safe; try {
// also allows subclasses to control the analyzer used. // reuse the same analyzer; it's thread-safe;
Analyzer analyzer = access.writer.getAnalyzer(); // also allows subclasses to control the analyzer used.
QueryParser parser = new QueryParser(Version.LUCENE_30, Analyzer analyzer = access.writer.getAnalyzer();
LUCENE_FIELD_DATA, analyzer); QueryParser parser = new QueryParser(Version.LUCENE_30,
Query query = parser.parse(text); LUCENE_FIELD_DATA, analyzer);
// Lucene 3 insists on a hard limit and will not provide Query query = parser.parse(text);
// a total hits value. Take at least 100 which is // Lucene 3 insists on a hard limit and will not provide
// an optimal limit for Lucene as any more // a total hits value. Take at least 100 which is
// will trigger writing results to disk. // an optimal limit for Lucene as any more
int maxResults = (limit == 0 ? 100 : limit) + offset; // will trigger writing results to disk.
TopDocs docs = searcher.search(query, maxResults); int maxResults = (limit == 0 ? 100 : limit) + offset;
if (limit == 0) { TopDocs docs = searcher.search(query, maxResults);
limit = docs.totalHits; if (limit == 0) {
} limit = docs.totalHits;
for (int i = 0, len = docs.scoreDocs.length; }
i < limit && i + offset < docs.totalHits for (int i = 0, len = docs.scoreDocs.length;
&& i + offset < len; i++) { i < limit && i + offset < docs.totalHits
ScoreDoc sd = docs.scoreDocs[i + offset]; && i + offset < len; i++) {
Document doc = searcher.doc(sd.doc); ScoreDoc sd = docs.scoreDocs[i + offset];
float score = sd.score; Document doc = searcher.doc(sd.doc);
String q = doc.get(LUCENE_FIELD_QUERY); float score = sd.score;
if (data) { String q = doc.get(LUCENE_FIELD_QUERY);
int idx = q.indexOf(" WHERE "); if (data) {
JdbcConnection c = (JdbcConnection) conn; int idx = q.indexOf(" WHERE ");
Session session = (Session) c.getSession(); JdbcConnection c = (JdbcConnection) conn;
Parser p = new Parser(session); Session session = (Session) c.getSession();
String tab = q.substring(0, idx); Parser p = new Parser(session);
ExpressionColumn expr = (ExpressionColumn) p.parseExpression(tab); String tab = q.substring(0, idx);
String schemaName = expr.getOriginalTableAliasName(); ExpressionColumn expr = (ExpressionColumn) p.parseExpression(tab);
String tableName = expr.getColumnName(); String schemaName = expr.getOriginalTableAliasName();
q = q.substring(idx + " WHERE ".length()); String tableName = expr.getColumnName();
Object[][] columnData = parseKey(conn, q); q = q.substring(idx + " WHERE ".length());
result.addRow( Object[][] columnData = parseKey(conn, q);
schemaName, result.addRow(
tableName, schemaName,
columnData[0], tableName,
columnData[1], columnData[0],
score); columnData[1],
} else { score);
result.addRow(q, score); } else {
result.addRow(q, score);
}
} }
} finally {
access.returnSearcher(searcher);
} }
} catch (Exception e) { } catch (Exception e) {
throw convertException(e); throw convertException(e);
...@@ -479,16 +469,16 @@ public class FullTextLucene extends FullText { ...@@ -479,16 +469,16 @@ public class FullTextLucene extends FullText {
/** /**
* Trigger updates the index when a inserting, updating, or deleting a row. * Trigger updates the index when a inserting, updating, or deleting a row.
*/ */
public static class FullTextTrigger implements Trigger { public static final class FullTextTrigger implements Trigger {
protected String schema; private String schema;
protected String table; private String table;
protected int[] keys; private int[] keys;
protected int[] indexColumns; private int[] indexColumns;
protected String[] columns; private String[] columns;
protected int[] columnTypes; private int[] columnTypes;
protected String indexPath; private String indexPath;
protected IndexAccess indexAccess; private IndexAccess indexAccess;
/** /**
* INTERNAL * INTERNAL
...@@ -541,9 +531,8 @@ public class FullTextLucene extends FullText { ...@@ -541,9 +531,8 @@ public class FullTextLucene extends FullText {
if (rs.next()) { if (rs.next()) {
String cols = rs.getString(1); String cols = rs.getString(1);
if (cols != null) { if (cols != null) {
for (String s : StringUtils.arraySplit(cols, ',', true)) { Collections.addAll(indexList,
indexList.add(s); StringUtils.arraySplit(cols, ',', true));
}
} }
} }
if (indexList.size() == 0) { if (indexList.size() == 0) {
...@@ -583,10 +572,7 @@ public class FullTextLucene extends FullText { ...@@ -583,10 +572,7 @@ public class FullTextLucene extends FullText {
*/ */
@Override @Override
public void close() throws SQLException { public void close() throws SQLException {
if (indexAccess != null) { removeIndexAccess(indexPath);
removeIndexAccess(indexAccess, indexPath);
indexAccess = null;
}
} }
/** /**
...@@ -600,14 +586,9 @@ public class FullTextLucene extends FullText { ...@@ -600,14 +586,9 @@ public class FullTextLucene extends FullText {
/** /**
* Commit all changes to the Lucene index. * Commit all changes to the Lucene index.
*/ */
void commitIndex() throws SQLException { private void commitIndex() throws SQLException {
try { try {
indexAccess.writer.commit(); indexAccess.commit();
// recreate Searcher with the IndexWriter's reader.
indexAccess.searcher.close();
indexAccess.reader.close();
indexAccess.reader = IndexReader.open(indexAccess.writer, true);
indexAccess.searcher = new IndexSearcher(indexAccess.reader);
} catch (IOException e) { } catch (IOException e) {
throw convertException(e); throw convertException(e);
} }
...@@ -699,22 +680,80 @@ public class FullTextLucene extends FullText { ...@@ -699,22 +680,80 @@ public class FullTextLucene extends FullText {
/** /**
* A wrapper for the Lucene writer and searcher. * A wrapper for the Lucene writer and searcher.
*/ */
static class IndexAccess { static final class IndexAccess {
/** /**
* The index writer. * The index writer.
*/ */
IndexWriter writer; final IndexWriter writer;
/** /**
* The index reader. * Map of usage counters for outstanding searchers.
*/ */
IndexReader reader; private final Map<IndexSearcher,Integer> counters = New.hashMap();
/**
* Usage counter for current searcher.
*/
private int counter;
/** /**
* The index searcher. * The index searcher.
*/ */
IndexSearcher searcher; private IndexSearcher searcher;
}
private IndexAccess(IndexWriter writer) throws IOException {
this.writer = writer;
IndexReader reader = IndexReader.open(writer, true);
searcher = new IndexSearcher(reader);
}
private synchronized IndexSearcher getSearcher() {
++counter;
return searcher;
}
private synchronized void returnSearcher(IndexSearcher searcher) {
if (this.searcher == searcher) {
--counter;
assert counter >= 0;
} else {
Integer cnt = counters.remove(searcher);
assert cnt != null;
if(--cnt == 0) {
closeSearcher(searcher);
} else {
counters.put(searcher, cnt);
}
}
}
public synchronized void commit() throws IOException {
writer.commit();
if (counter != 0) {
counters.put(searcher, counter);
counter = 0;
} else {
closeSearcher(searcher);
}
// recreate Searcher with the IndexWriter's reader.
searcher = new IndexSearcher(IndexReader.open(writer, true));
}
public synchronized void close() throws IOException {
for (IndexSearcher searcher : counters.keySet()) {
closeSearcher(searcher);
}
counters.clear();
closeSearcher(searcher);
searcher = null;
writer.close();
}
private static void closeSearcher(IndexSearcher searcher) {
IndexReader indexReader = searcher.getIndexReader();
try { searcher.close(); } catch(IOException ignore) {/**/}
try { indexReader.close(); } catch(IOException ignore) {/**/}
}
}
} }
h2/src/main/org/h2/fulltext/FullTextSettings.java
浏览文件 @ e1a91d29
...@@ -10,20 +10,22 @@ import java.sql.PreparedStatement; ...@@ -10,20 +10,22 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.HashMap; import java.util.Collections;
import java.util.HashSet; import java.util.Map;
import java.util.Set;
import org.h2.util.New; import org.h2.util.New;
import org.h2.util.SoftHashMap; import org.h2.util.SoftHashMap;
/** /**
* The global settings of a full text search. * The global settings of a full text search.
*/ */
class FullTextSettings { final class FullTextSettings {
/** /**
* The settings of open indexes. * The settings of open indexes.
*/ */
private static final HashMap<String, FullTextSettings> SETTINGS = New.hashMap(); private static final Map<String, FullTextSettings> SETTINGS = Collections.synchronizedMap(New.<String, FullTextSettings>hashMap());
/** /**
* Whether this instance has been initialized. * Whether this instance has been initialized.
...@@ -33,17 +35,17 @@ class FullTextSettings { ...@@ -33,17 +35,17 @@ class FullTextSettings {
/** /**
* The set of words not to index (stop words). * The set of words not to index (stop words).
*/ */
private final HashSet<String> ignoreList = New.hashSet(); private final Set<String> ignoreList = Collections.synchronizedSet(New.<String>hashSet());
/** /**
* The set of words / terms. * The set of words / terms.
*/ */
private final HashMap<String, Integer> words = New.hashMap(); private final Map<String, Integer> words = Collections.synchronizedMap(New.<String, Integer>hashMap());
/** /**
* The set of indexes in this database. * The set of indexes in this database.
*/ */
private final HashMap<Integer, IndexInfo> indexes = New.hashMap(); private final Map<Integer, IndexInfo> indexes = Collections.synchronizedMap(New.<Integer, IndexInfo>hashMap());
/** /**
* The prepared statement cache. * The prepared statement cache.
...@@ -60,7 +62,7 @@ class FullTextSettings { ...@@ -60,7 +62,7 @@ class FullTextSettings {
/** /**
* Create a new instance. * Create a new instance.
*/ */
protected FullTextSettings() { private FullTextSettings() {
// don't allow construction // don't allow construction
} }
...@@ -69,7 +71,7 @@ class FullTextSettings { ...@@ -69,7 +71,7 @@ class FullTextSettings {
* *
* @return the ignore list * @return the ignore list
*/ */
protected HashSet<String> getIgnoreList() { protected Set<String> getIgnoreList() {
return ignoreList; return ignoreList;
} }
...@@ -78,7 +80,7 @@ class FullTextSettings { ...@@ -78,7 +80,7 @@ class FullTextSettings {
* *
* @return the word list * @return the word list
*/ */
protected HashMap<String, Integer> getWordList() { protected Map<String, Integer> getWordList() {
return words; return words;
} }
...@@ -140,7 +142,7 @@ class FullTextSettings { ...@@ -140,7 +142,7 @@ class FullTextSettings {
* @param conn the connection * @param conn the connection
* @return the file system path * @return the file system path
*/ */
protected static String getIndexPath(Connection conn) throws SQLException { private static String getIndexPath(Connection conn) throws SQLException {
Statement stat = conn.createStatement(); Statement stat = conn.createStatement();
ResultSet rs = stat.executeQuery( ResultSet rs = stat.executeQuery(
"CALL IFNULL(DATABASE_PATH(), 'MEM:' || DATABASE())"); "CALL IFNULL(DATABASE_PATH(), 'MEM:' || DATABASE())");
......
h2/src/test/org/h2/test/db/TestFullText.java
浏览文件 @ e1a91d29
...@@ -50,10 +50,6 @@ public class TestFullText extends TestBase { ...@@ -50,10 +50,6 @@ public class TestFullText extends TestBase {
@Override @Override
public void test() throws Exception { public void test() throws Exception {
if (config.multiThreaded) {
// It is even mentioned in the docs that this is not supported
return;
}
testUuidPrimaryKey(false); testUuidPrimaryKey(false);
testAutoAnalyze(); testAutoAnalyze();
testNativeFeatures(); testNativeFeatures();
...@@ -71,7 +67,9 @@ public class TestFullText extends TestBase { ...@@ -71,7 +67,9 @@ public class TestFullText extends TestBase {
testCreateDropLucene(); testCreateDropLucene();
testUuidPrimaryKey(true); testUuidPrimaryKey(true);
testMultiThreaded(true); testMultiThreaded(true);
testMultiThreaded(false); if(config.mvStore || !config.multiThreaded) {
testMultiThreaded(false);
}
testTransaction(true); testTransaction(true);
test(true, "VARCHAR"); test(true, "VARCHAR");
test(true, "CLOB"); test(true, "CLOB");
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论