Skip to content

H
h2database
提交 ff4d721e authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

Page store: opening a large database was slow if it was not closed before.

上级 2460e391
隐藏空白字符变更
内嵌 并排
正在显示 包含 184 行增加 和 46 行删除
h2/src/docsrc/html/changelog.html
浏览文件 @ ff4d721e
......@@ -18,7 +18,8 @@ Change Log
<h1>Change Log</h1>
<h2>Next Version (unreleased)</h2>
<ul><li>Page store: new write and read counters in the meta data table. Use
<ul><li>Page store: opening a large database was slow if it was not closed before.
</li><li>Page store: new write and read counters in the meta data table. Use
SELECT * FROM INFORMATION_SCHEMA.SETTINGS WHERE NAME IN(
'info.FILE_WRITE_TOTAL', 'info.FILE_WRITE', 'info.FILE_READ',
'info.CACHE_MAX_SIZE', 'info.CACHE_SIZE')
......
h2/src/main/org/h2/api/DatabaseEventListener.java
浏览文件 @ ff4d721e
......@@ -18,7 +18,7 @@ import java.util.EventListener;
public interface DatabaseEventListener extends EventListener {
/**
* This state is used when scanning the data or index file.
* This state is used when scanning the database file.
*/
int STATE_SCAN_FILE = 0;
......
h2/src/main/org/h2/index/PageBtree.java
浏览文件 @ ff4d721e
......@@ -89,7 +89,6 @@ public abstract class PageBtree extends Page {
*
* @param rowCount the stored row count
*/
// TODO remove
abstract void setRowCountStored(int rowCount) throws SQLException;
/**
......
h2/src/main/org/h2/index/PageBtreeIndex.java
浏览文件 @ ff4d721e
......@@ -18,6 +18,7 @@ import org.h2.store.PageStore;
import org.h2.table.Column;
import org.h2.table.IndexColumn;
import org.h2.table.TableData;
import org.h2.util.MathUtils;
import org.h2.value.Value;
import org.h2.value.ValueLob;
import org.h2.value.ValueNull;
......@@ -100,6 +101,7 @@ public class PageBtreeIndex extends PageIndex {
store.update(newRoot);
root = newRoot;
}
invalidateRowCount();
rowCount++;
}
......@@ -217,6 +219,7 @@ public class PageBtreeIndex extends PageIndex {
} else {
PageBtree root = getPage(rootPageId);
root.remove(row);
invalidateRowCount();
rowCount--;
}
}
......@@ -275,14 +278,16 @@ public class PageBtreeIndex extends PageIndex {
}
public long getRowCount(Session session) {
return tableData.getRowCount(session);
return rowCount;
}
public void close(Session session) {
public void close(Session session) throws SQLException {
if (trace.isDebugEnabled()) {
trace.debug("close");
}
// TODO write the row count
// can not close the index because it might get used afterwards,
// for example after running recovery
writeRowCount();
}
/**
......@@ -363,4 +368,14 @@ public class PageBtreeIndex extends PageIndex {
store.addIndex(this);
}
private void invalidateRowCount() throws SQLException {
PageBtree root = getPage(rootPageId);
root.setRowCountStored(PageData.UNKNOWN_ROWCOUNT);
}
public void writeRowCount() throws SQLException {
PageBtree root = getPage(rootPageId);
root.setRowCountStored(MathUtils.convertLongToInt(rowCount));
}
}
h2/src/main/org/h2/index/PageBtreeNode.java
浏览文件 @ ff4d721e
......@@ -7,6 +7,7 @@
package org.h2.index;
import java.sql.SQLException;
import org.h2.api.DatabaseEventListener;
import org.h2.constant.ErrorCode;
import org.h2.constant.SysProperties;
import org.h2.engine.Session;
......@@ -41,7 +42,7 @@ public class PageBtreeNode extends PageBtree {
*/
private int[] childPageIds;
// private int rowCountStored = UNKNOWN_ROWCOUNT;
private int rowCountStored = UNKNOWN_ROWCOUNT;
private int rowCount = UNKNOWN_ROWCOUNT;
......@@ -93,11 +94,8 @@ public class PageBtreeNode extends PageBtree {
"page:" + getPos() + " expected index:" + index.getId() +
"got:" + indexId);
}
rowCount = data.readInt();
rowCount = rowCountStored = data.readInt();
entryCount = data.readShortInt();
if (!PageStore.STORE_BTREE_ROWCOUNT) {
rowCount = UNKNOWN_ROWCOUNT;
}
childPageIds = new int[entryCount + 1];
childPageIds[entryCount] = data.readInt();
rows = PageStore.newSearchRows(entryCount);
......@@ -213,10 +211,18 @@ public class PageBtreeNode extends PageBtree {
return -1;
}
private void updateRowCount(int offset) {
if (PageStore.STORE_BTREE_ROWCOUNT) {
private void updateRowCount(int offset) throws SQLException {
if (rowCount != UNKNOWN_ROWCOUNT) {
rowCount += offset;
}
if (rowCountStored != UNKNOWN_ROWCOUNT) {
rowCountStored = UNKNOWN_ROWCOUNT;
index.getPageStore().logUndo(this, data);
if (written) {
writeHead();
}
index.getPageStore().update(this);
}
}
PageBtree split(int splitPoint) throws SQLException {
......@@ -350,14 +356,23 @@ public class PageBtreeNode extends PageBtree {
for (int child : childPageIds) {
PageBtree page = index.getPage(child);
count += page.getRowCount();
index.getDatabase().setProgress(DatabaseEventListener.STATE_SCAN_FILE, index.getName(), count, Integer.MAX_VALUE);
}
rowCount = count;
}
return rowCount;
}
void setRowCountStored(int rowCount) {
void setRowCountStored(int rowCount) throws SQLException {
this.rowCount = rowCount;
if (rowCountStored != rowCount) {
rowCountStored = rowCount;
index.getPageStore().logUndo(this, data);
if (written) {
writeHead();
}
index.getPageStore().update(this);
}
}
private void check() {
......@@ -384,7 +399,7 @@ public class PageBtreeNode extends PageBtree {
data.writeShortInt(0);
data.writeInt(parentPageId);
data.writeVarInt(index.getId());
data.writeInt(rowCount);
data.writeInt(rowCountStored);
data.writeShortInt(entryCount);
}
......@@ -510,6 +525,8 @@ public class PageBtreeNode extends PageBtree {
store.logUndo(this, data);
PageBtreeNode p2 = PageBtreeNode.create(index, newPos, parentPageId);
readAllRows();
p2.rowCountStored = rowCountStored;
p2.rowCount = rowCount;
p2.childPageIds = childPageIds;
p2.rows = rows;
p2.entryCount = entryCount;
......
h2/src/main/org/h2/index/PageDataIndex.java
浏览文件 @ ff4d721e
......@@ -307,11 +307,6 @@ public class PageDataIndex extends PageIndex implements RowIndex {
store.logAddOrRemoveRow(session, tableData.getId(), row, false);
}
private void invalidateRowCount() throws SQLException {
PageData root = getPage(rootPageId, 0);
root.setRowCountStored(PageData.UNKNOWN_ROWCOUNT);
}
public void remove(Session session) throws SQLException {
if (trace.isDebugEnabled()) {
trace.debug(this + " remove");
......@@ -424,8 +419,7 @@ public class PageDataIndex extends PageIndex implements RowIndex {
}
// can not close the index because it might get used afterwards,
// for example after running recovery
PageData root = getPage(rootPageId, 0);
root.setRowCountStored(MathUtils.convertLongToInt(rowCount));
writeRowCount();
}
Iterator<Row> getDelta() {
......@@ -484,4 +478,14 @@ public class PageDataIndex extends PageIndex implements RowIndex {
return getName();
}
private void invalidateRowCount() throws SQLException {
PageData root = getPage(rootPageId, 0);
root.setRowCountStored(PageData.UNKNOWN_ROWCOUNT);
}
public void writeRowCount() throws SQLException {
PageData root = getPage(rootPageId, 0);
root.setRowCountStored(MathUtils.convertLongToInt(rowCount));
}
}
h2/src/main/org/h2/index/PageDataNode.java
浏览文件 @ ff4d721e
......@@ -8,6 +8,7 @@ package org.h2.index;
import java.sql.SQLException;
import java.util.Arrays;
import org.h2.api.DatabaseEventListener;
import org.h2.constant.ErrorCode;
import org.h2.constant.SysProperties;
import org.h2.engine.Session;
......@@ -292,6 +293,7 @@ public class PageDataNode extends PageData {
throw Message.throwInternalError("Page it its own child: " + getPos());
}
count += page.getRowCount();
index.getDatabase().setProgress(DatabaseEventListener.STATE_SCAN_FILE, index.getTable() + "." + index.getName(), count, Integer.MAX_VALUE);
}
rowCount = count;
}
......
h2/src/main/org/h2/index/PageDelegateIndex.java
浏览文件 @ ff4d721e
......@@ -112,4 +112,8 @@ public class PageDelegateIndex extends PageIndex {
return mainIndex.getRowCountApproximation();
}
public void writeRowCount() {
// ignore
}
}
h2/src/main/org/h2/index/PageIndex.java
浏览文件 @ ff4d721e
......@@ -6,6 +6,8 @@
*/
package org.h2.index;
import java.sql.SQLException;
/**
* A page store index.
*/
......@@ -24,4 +26,9 @@ public abstract class PageIndex extends BaseIndex {
return 0;
}
/**
* Write back the row count if it has changed.
*/
public abstract void writeRowCount() throws SQLException;
}
h2/src/main/org/h2/store/PageStore.java
浏览文件 @ ff4d721e
......@@ -90,21 +90,18 @@ public class PageStore implements CacheWriter {
// TODO optimization: check if calling Data.getValueLen slows things down
// TODO order pages so that searching for a key only seeks forward
// TODO optimization: update: only log the key and changed values
// TODO maybe remove some parent pointers
// TODO index creation: use less space (ordered, split at insertion point)
// TODO detect circles in linked lists
// (input stream, free list, extend pages...)
// at runtime and recovery
// synchronized correctly (on the index?)
// TODO remove trace or use isDebugEnabled
// TODO recover tool: don't re-do uncommitted operations
// TODO recover tool: support syntax to delete a row with a key
// TODO don't store default values (store a special value)
// TODO split files (1 GB max size)
// TODO add a setting (that can be changed at runtime) to call fsync
// and delay on each commit
// TODO check for file size (exception if not exact size expected)
// TODO implement missing code for STORE_BTREE_ROWCOUNT (maybe enable)
// TODO online backup using bsdiff
// TODO when removing DiskFile:
......@@ -132,11 +129,6 @@ public class PageStore implements CacheWriter {
*/
public static final int PAGE_SIZE_DEFAULT = 2 * 1024;
/**
* Store the rowcount in b-tree indexes.
*/
public static final boolean STORE_BTREE_ROWCOUNT = false;
private static final int PAGE_ID_FREE_LIST_ROOT = 3;
private static final int PAGE_ID_META_ROOT = 4;
......@@ -188,7 +180,7 @@ public class PageStore implements CacheWriter {
private TableData metaTable;
private PageDataIndex metaIndex;
private IntIntHashMap metaRootPageId = new IntIntHashMap();
private HashMap<Integer, Index> metaObjects = New.hashMap();
private HashMap<Integer, PageIndex> metaObjects = New.hashMap();
/**
* The map of reserved pages, to ensure index head pages
......@@ -313,6 +305,12 @@ public class PageStore implements CacheWriter {
}
}
private void writeIndexRowCounts() throws SQLException {
for (PageIndex index: metaObjects.values()) {
index.writeRowCount();
}
}
private void writeBack() throws SQLException {
ObjectArray<CacheObject> list = cache.getAllChanged();
CacheObject.sort(list);
......@@ -332,6 +330,7 @@ public class PageStore implements CacheWriter {
}
synchronized (database) {
database.checkPowerOff();
writeIndexRowCounts();
writeBack();
log.checkpoint();
switchLog();
......@@ -1034,10 +1033,13 @@ public class PageStore implements CacheWriter {
}
recoveryRunning = false;
reservedPages = null;
writeIndexRowCounts();
writeBack();
// clear the cache because it contains pages with closed indexes
cache.clear();
freeLists.clear();
metaObjects.clear();
metaObjects.put(-1, metaIndex);
if (setReadOnly) {
database.setReadOnly(true);
}
......@@ -1201,12 +1203,9 @@ public class PageStore implements CacheWriter {
private void removeMeta(int logPos, Row row) throws SQLException {
int id = row.getValue(0).getInt();
Index index = metaObjects.get(id);
PageIndex index = metaObjects.get(id);
int rootPageId = index.getRootPageId();
index.getTable().removeIndex(index);
if (index instanceof MultiVersionIndex) {
index = ((MultiVersionIndex) index).getBaseIndex();
}
if (index instanceof PageBtreeIndex) {
if (index.isTemporary()) {
systemSession.removeLocalTempTableIndex(index);
......@@ -1297,7 +1296,13 @@ public class PageStore implements CacheWriter {
}
meta = table.addIndex(session, "I" + id, id, cols, indexType, id, null);
}
metaObjects.put(id, meta);
PageIndex index;
if (meta instanceof MultiVersionIndex) {
index = (PageIndex) ((MultiVersionIndex) meta).getBaseIndex();
} else {
index = (PageIndex) meta;
}
metaObjects.put(id, index);
}
/**
......
h2/src/main/org/h2/table/TableData.java
浏览文件 @ ff4d721e
......@@ -10,7 +10,6 @@ import java.sql.SQLException;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import org.h2.api.DatabaseEventListener;
import org.h2.command.ddl.CreateTableData;
import org.h2.constant.ErrorCode;
......@@ -28,8 +27,8 @@ import org.h2.index.IndexType;
import org.h2.index.MultiVersionIndex;
import org.h2.index.NonUniqueHashIndex;
import org.h2.index.PageBtreeIndex;
import org.h2.index.PageDelegateIndex;
import org.h2.index.PageDataIndex;
import org.h2.index.PageDelegateIndex;
import org.h2.index.RowIndex;
import org.h2.index.ScanIndex;
import org.h2.index.TreeIndex;
......@@ -39,7 +38,6 @@ import org.h2.result.Row;
import org.h2.result.SortOrder;
import org.h2.schema.SchemaObject;
import org.h2.store.DataPage;
import org.h2.store.PageStore;
import org.h2.store.Record;
import org.h2.store.RecordReader;
import org.h2.util.MathUtils;
......@@ -156,12 +154,11 @@ public class TableData extends Table implements RecordReader {
private void checkRowCount(Session session, Index index, int offset) {
if (SysProperties.CHECK && !database.isMultiVersion()) {
if (database.isPageStoreEnabled() && !PageStore.STORE_BTREE_ROWCOUNT) {
return;
}
long rc = index.getRowCount(session);
if (rc != rowCount + offset) {
Message.throwInternalError("rowCount expected " + (rowCount + offset) + " got " + rc + " " + getName() + "." + index.getName());
if (!(index instanceof PageDelegateIndex)) {
long rc = index.getRowCount(session);
if (rc != rowCount + offset) {
Message.throwInternalError("rowCount expected " + (rowCount + offset) + " got " + rc + " " + getName() + "." + index.getName());
}
}
}
}
......
h2/src/test/org/h2/test/unit/TestPageStore.java
浏览文件 @ ff4d721e
......@@ -15,12 +15,15 @@ import java.sql.Statement;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import org.h2.api.DatabaseEventListener;
import org.h2.test.TestBase;
/**
* Test the page store.
*/
public class TestPageStore extends TestBase {
public class TestPageStore extends TestBase implements DatabaseEventListener {
static StringBuilder eventBuffer = new StringBuilder();
/**
* Run just this test.
......@@ -33,6 +36,8 @@ public class TestPageStore extends TestBase {
}
public void test() throws Exception {
testLargeDatabaseFastOpen();
testUniqueIndexReopen();
testExistingOld();
testLargeRows();
testRecoverDropIndex();
......@@ -45,6 +50,56 @@ public class TestPageStore extends TestBase {
testFuzzOperations();
}
private void testLargeDatabaseFastOpen() throws SQLException {
if (config.memory) {
return;
}
deleteDb("pageStore");
Connection conn;
String url = "pageStore";
conn = getConnection(url);
conn.createStatement().execute("CREATE TABLE TEST(ID INT PRIMARY KEY, NAME VARCHAR)");
conn.createStatement().execute("create unique index idx_test_name on test(name)");
conn.createStatement().execute("INSERT INTO TEST SELECT X, X || space(10) FROM SYSTEM_RANGE(1, 1000)");
conn.close();
conn = getConnection(url);
conn.createStatement().execute("DELETE FROM TEST WHERE ID=1");
conn.createStatement().execute("CHECKPOINT");
conn.createStatement().execute("SHUTDOWN IMMEDIATELY");
try {
conn.close();
} catch (SQLException e) {
// ignore
}
eventBuffer.setLength(0);
conn = getConnection(url + ";DATABASE_EVENT_LISTENER='" + getClass().getName() + "'");
assertEquals("init;opened;", eventBuffer.toString());
conn.close();
}
private void testUniqueIndexReopen() throws SQLException {
if (config.memory) {
return;
}
deleteDb("pageStore");
Connection conn;
String url = "pageStore";
conn = getConnection(url);
conn.createStatement().execute("CREATE TABLE test(ID INT PRIMARY KEY, NAME VARCHAR(255))");
conn.createStatement().execute("create unique index idx_test_name on test(name)");
conn.createStatement().execute("INSERT INTO TEST VALUES(1, 'Hello')");
conn.close();
conn = getConnection(url);
try {
conn.createStatement().execute("INSERT INTO TEST VALUES(2, 'Hello')");
fail();
} catch (SQLException e) {
assertKnownException(e);
}
conn.close();
}
private void testExistingOld() throws SQLException {
if (config.memory) {
return;
......@@ -359,4 +414,36 @@ public class TestPageStore extends TestBase {
trace(" " + m);
}
public void closingDatabase() {
event("closing");
}
public void diskSpaceIsLow(long stillAvailable) {
event("diskSpaceIsLow " + stillAvailable);
}
public void exceptionThrown(SQLException e, String sql) {
event("exceptionThrown " + e + " " + sql);
}
public void init(String url) {
event("init");
}
public void opened() {
event("opened");
}
public void setProgress(int state, String name, int x, int max) {
if (name.startsWith("SYS:SYS_ID")) {
// ignore
return;
}
event("setProgress " + state + " " + name + " " + x + " " + max);
}
private void event(String s) {
eventBuffer.append(s).append(';');
}
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论