提交 50e52985 authored 作者: Thomas Mueller's avatar Thomas Mueller

Page store: reduce database size.

上级 d0f7074a
......@@ -7,7 +7,11 @@
package org.h2.index;
/**
* A page.
* A page. Format:
* <ul><li>0-3: parent page id (0 for root)
* </li><li>4-4: page type
* </li><li>page-type specific data
* </li></ul>
*/
public class Page {
......
......@@ -60,17 +60,21 @@ class PageBtreeLeaf extends PageBtree {
int last = entryCount == 0 ? pageSize : offsets[entryCount - 1];
if (last - rowLength < start + OFFSET_LENGTH) {
if (entryCount > 1) {
return entryCount / 2;
// split at the insertion point to better fill pages
// split in half would be:
// return entryCount / 2;
int x = find(row, false, true, true);
return x < 2 ? 2 : x >= entryCount - 3 ? entryCount - 3 : x;
}
onlyPosition = true;
// change the offsets (now storing only positions)
int o = pageSize;
for (int i = 0; i < entryCount; i++) {
o -= index.getRowSize(data, getRow(i), onlyPosition);
o -= index.getRowSize(data, getRow(i), true);
offsets[i] = o;
}
last = entryCount == 0 ? pageSize : offsets[entryCount - 1];
rowLength = index.getRowSize(data, row, onlyPosition);
rowLength = index.getRowSize(data, row, true);
if (SysProperties.CHECK && last - rowLength < start + OFFSET_LENGTH) {
throw Message.throwInternalError();
}
......
......@@ -105,11 +105,11 @@ class PageBtreeNode extends PageBtree {
// change the offsets (now storing only positions)
int o = pageSize;
for (int i = 0; i < entryCount; i++) {
o -= index.getRowSize(data, getRow(i), onlyPosition);
o -= index.getRowSize(data, getRow(i), true);
offsets[i] = o;
}
last = entryCount == 0 ? pageSize : offsets[entryCount - 1];
rowLength = index.getRowSize(data, row, onlyPosition);
rowLength = index.getRowSize(data, row, true);
if (SysProperties.CHECK && last - rowLength < start + CHILD_OFFSET_PAIR_LENGTH) {
throw Message.throwInternalError();
}
......
......@@ -97,9 +97,11 @@ class PageDataLeaf extends PageData {
int pageSize = index.getPageStore().getPageSize();
int last = entryCount == 0 ? pageSize : offsets[entryCount - 1];
if (entryCount > 0 && last - rowLength < start + KEY_OFFSET_PAIR_LENGTH) {
if (entryCount > 1) {
return entryCount / 2;
}
// split at the insertion point to better fill pages
// split in half would be:
// if (entryCount > 1) {
// return entryCount / 2;
// }
return find(row.getPos());
}
int offset = last - rowLength;
......
......@@ -19,6 +19,6 @@ org.h2.tools.RunScript.main=Options are case sensitive. Supported options are\:\
org.h2.tools.Script=Creates a SQL script file by extracting the schema and data of a database.
org.h2.tools.Script.main=Options are case sensitive. Supported options are\:\n[-help] or [-?] Print the list of options\n[-url "<url>"] The database URL (jdbc\:...)\n[-user <user>] The user name (default\: sa)\n[-password <pwd>] The password\n[-script <file>] The target script file name (default\: backup.sql)\n[-options ...] A list of options (only for embedded H2, see RUNSCRIPT)\n[-quiet] Do not print progress information
org.h2.tools.Server=Starts the H2 Console (web-) server, TCP, and PG server.
org.h2.tools.Server.main=When running without options, -tcp, -web, -browser and -pg are started.\nOptions are case sensitive. Supported options are\:\n[-help] or [-?] Print the list of options\n[-web] Start the web server with the H2 Console\n[-webAllowOthers] Allow other computers to connect\n[-webPort <port>] The port (default\: 8082)\n[-webSSL] Use encrypted (HTTPS) connections\n[-browser] Start a browser and open a page to connect to the web server\n[-tcp] Start the TCP server\n[-tcpAllowOthers] Allow other computers to connect\n[-tcpPort <port>] The port (default\: 9092)\n[-tcpSSL] Use encrypted (SSL) connections\n[-tcpPassword <pwd>] The password for shutting down a TCP server\n[-tcpShutdown "<url>"] Stop the TCP server; example\: tcp\://localhost\:9094\n[-tcpShutdownForce] Do not wait until all connections are closed\n[-pg] Start the PG server\n[-pgAllowOthers] Allow other computers to connect\n[-pgPort <port>] The port (default\: 5435)\n[-baseDir <dir>] The base directory for H2 databases; for all servers\n[-ifExists] Only existing databases may be opened; for all servers\n[-trace] Print additional trace information; for all servers
org.h2.tools.Server.main=When running without options, -tcp, -web, -browser and -pg are started.\nOptions are case sensitive. Supported options are\:\n[-help] or [-?] Print the list of options\n[-web] Start the web server with the H2 Console\n[-webAllowOthers] Allow other computers to connect - see below\n[-webPort <port>] The port (default\: 8082)\n[-webSSL] Use encrypted (HTTPS) connections\n[-browser] Start a browser and open a page to connect to the web server\n[-tcp] Start the TCP server\n[-tcpAllowOthers] Allow other computers to connect - see below\n[-tcpPort <port>] The port (default\: 9092)\n[-tcpSSL] Use encrypted (SSL) connections\n[-tcpPassword <pwd>] The password for shutting down a TCP server\n[-tcpShutdown "<url>"] Stop the TCP server; example\: tcp\://localhost\:9094\n[-tcpShutdownForce] Do not wait until all connections are closed\n[-pg] Start the PG server\n[-pgAllowOthers] Allow other computers to connect - see below\n[-pgPort <port>] The port (default\: 5435)\n[-baseDir <dir>] The base directory for H2 databases; for all servers\n[-ifExists] Only existing databases may be opened; for all servers\n[-trace] Print additional trace information; for all servers\nThe options -xAllowOthers are potentially risky.\nFor details, see Advanced Topics / Protection against Remote Access.
org.h2.tools.Shell=Interactive command line tool to access a database using JDBC.
org.h2.tools.Shell.main=Options are case sensitive. Supported options are\:\n[-help] or [-?] Print the list of options\n[-url "<url>"] The database URL (jdbc\:h2\:...)\n[-user <user>] The user name\n[-password <pwd>] The password\n[-driver <class>] The JDBC driver class to use (not required in most cases)\nIf special characters don't work as expected, you may need to use\n -Dfile.encoding\=UTF-8 (Mac OS X) or CP850 (Windows).
......@@ -69,6 +69,22 @@ public class PageFreeList extends Record {
}
}
int getFirstFree() {
if (full) {
return -1;
}
int free = used.nextClearBit(0);
if (free >= pageCount) {
return -1;
}
return free;
}
int getLastUsed() {
int last = used.getLastSetBit();
return last == -1 ? -1 : last + getPos();
}
/**
* Mark a page as used.
*
......
......@@ -18,6 +18,7 @@ import org.h2.engine.Session;
import org.h2.index.Cursor;
import org.h2.index.Index;
import org.h2.index.IndexType;
import org.h2.index.Page;
import org.h2.index.PageBtreeIndex;
import org.h2.index.PageScanIndex;
import org.h2.log.InDoubtTransaction;
......@@ -71,17 +72,22 @@ import org.h2.value.ValueString;
*/
public class PageStore implements CacheWriter {
// TODO var int: see google protocol buffers
// TODO don't save parent (only root); remove setPageId
// TODO implement checksum - 0 for empty
// TODO b-tree index with fixed size values doesn't need offset and so on
// TODO shrinking: a way to load pages centrally
// TODO shrinking: Page.moveTo(int pageId).
// TODO utf-x: test if it's faster
// TODO value serialization: test (100% coverage)
// TODO scan index: support long keys, and use var long
// TODO don't save the direct parent (only root); remove setPageId
// TODO implement checksum; 0 for empty pages
// TODO remove parent, use tableId if required
// TODO replace CRC32
// TODO optimization: try to avoid allocating a byte array per page
// TODO optimization: check if calling Data.getValueLen slows things down
// TODO PageBtreeNode: 4 bytes offset - others use only 2
// TODO block compression: don't store the middle zeroes
// TODO block compression: maybe http://en.wikipedia.org/wiki/LZJB
// with RLE, specially for 0s.
// TODO undo pages: don't store the middle zeroes
// TODO undo pages compression: try http://en.wikipedia.org/wiki/LZJB
// TODO order pages so that searching for a key only seeks forward
// TODO completely re-use keys of deleted rows; maybe
// remember last page with deleted keys (in the root page?),
......@@ -341,13 +347,62 @@ public class PageStore implements CacheWriter {
writeCount++;
}
}
// TODO shrink file if required here
// int pageCount = getFreeList().getLastUsed() + 1;
// trace.debug("pageCount:" + pageCount);
// file.setLength((long) pageCount << pageSizeShift);
}
}
private void compact() throws SQLException {
trim();
int full = pageCount - 1;
int free = -1;
for (int i = 0; i < pageCount && free != -1; i++) {
free = getFreeList(i).getFirstFree();
}
if (free == -1 || free < 10) {
return;
}
Record rec = getRecord(full);
if (rec == null) {
} else {
Data page = Data.create(database, pageSize);
readPage(full, page);
int parent = page.readInt();
int type = page.readByte();
boolean last = (type & Page.FLAG_LAST) != 0;
type = type & ~Page.FLAG_LAST;
System.out.println("last page is " + type + " parent: " + parent);
switch (type) {
case Page.TYPE_EMPTY:
break;
case Page.TYPE_FREE_LIST:
break;
case Page.TYPE_DATA_LEAF:
break;
case Page.TYPE_DATA_NODE:
case Page.TYPE_DATA_OVERFLOW:
case Page.TYPE_BTREE_LEAF:
case Page.TYPE_BTREE_NODE:
case Page.TYPE_STREAM_TRUNK:
case Page.TYPE_STREAM_DATA:
}
}
}
/**
* Shrink the file so there are no empty pages at the end.
*/
public void trim() throws SQLException {
for (int i = getFreeListId(pageCount); i >= 0; i--) {
int last = getFreeList(i).getLastUsed();
if (last != -1) {
pageCount = last + 1;
break;
}
}
trace.debug("pageCount:" + pageCount);
file.setLength((long) pageCount << pageSizeShift);
}
private void switchLog() throws SQLException {
trace.debug("switchLog");
Session[] sessions = database.getSessions(true);
......@@ -540,8 +595,12 @@ public class PageStore implements CacheWriter {
}
}
private int getFreeListId(int pageId) {
return (pageId - PAGE_ID_FREE_LIST_ROOT) / freeListPagesPerList;
}
private PageFreeList getFreeListForPage(int pageId) throws SQLException {
return getFreeList((pageId - PAGE_ID_FREE_LIST_ROOT) / freeListPagesPerList);
return getFreeList(getFreeListId(pageId));
}
private PageFreeList getFreeList(int i) throws SQLException {
......
......@@ -720,6 +720,7 @@ public class Recover extends Tool implements DataHandler {
setDatabaseName(fileName.substring(0, fileName.length() - Constants.SUFFIX_PAGE_FILE.length()));
FileStore store = null;
PrintWriter writer = null;
int[] pageTypeCount = new int[Page.TYPE_STREAM_DATA + 2];
int emptyPages = 0;
try {
writer = getWriter(fileName, ".sql");
......@@ -789,6 +790,7 @@ public class Recover extends Tool implements DataHandler {
int type = s.readByte();
switch (type) {
case Page.TYPE_EMPTY:
pageTypeCount[type]++;
if (parentPageId != 0) {
writer.println("-- ERROR empty page with parent: " + parentPageId);
}
......@@ -798,29 +800,31 @@ public class Recover extends Tool implements DataHandler {
boolean last = (type & Page.FLAG_LAST) != 0;
type &= ~Page.FLAG_LAST;
switch (type) {
case Page.TYPE_DATA_OVERFLOW:
writer.println("-- page " + page + ": data overflow " + (last ? "(last)" : ""));
break;
case Page.TYPE_DATA_NODE: {
int entries = s.readShortInt();
int rowCount = s.readInt();
writer.println("-- page " + page + ": data node " + (last ? "(last)" : "") + " entries: " + entries + " rowCount: " + rowCount);
break;
}
// type 1
case Page.TYPE_DATA_LEAF: {
pageTypeCount[type]++;
setStorage(s.readInt());
int entries = s.readShortInt();
writer.println("-- page " + page + ": data leaf " + (last ? "(last)" : "") + " table: " + storageId + " entries: " + entries);
dumpPageDataLeaf(store, pageSize, writer, s, last, page, entries);
break;
}
case Page.TYPE_BTREE_NODE:
writer.println("-- page " + page + ": b-tree node" + (last ? "(last)" : ""));
if (trace) {
dumpPageBtreeNode(writer, s, !last);
// type 2
case Page.TYPE_DATA_NODE: {
pageTypeCount[type]++;
int entries = s.readShortInt();
int rowCount = s.readInt();
writer.println("-- page " + page + ": data node " + (last ? "(last)" : "") + " entries: " + entries + " rowCount: " + rowCount);
break;
}
// type 3
case Page.TYPE_DATA_OVERFLOW:
pageTypeCount[type]++;
writer.println("-- page " + page + ": data overflow " + (last ? "(last)" : ""));
break;
// type 4
case Page.TYPE_BTREE_LEAF: {
pageTypeCount[type]++;
setStorage(s.readInt());
int entries = s.readShortInt();
writer.println("-- page " + page + ": b-tree leaf " + (last ? "(last)" : "") + " table: " + storageId + " entries: " + entries);
......@@ -829,14 +833,28 @@ public class Recover extends Tool implements DataHandler {
}
break;
}
// type 5
case Page.TYPE_BTREE_NODE:
pageTypeCount[type]++;
writer.println("-- page " + page + ": b-tree node" + (last ? "(last)" : ""));
if (trace) {
dumpPageBtreeNode(writer, s, !last);
}
break;
// type 6
case Page.TYPE_FREE_LIST:
pageTypeCount[type]++;
writer.println("-- page " + page + ": free list " + (last ? "(last)" : ""));
free += dumpPageFreeList(writer, s, pageSize, page, pageCount);
break;
// type 7
case Page.TYPE_STREAM_TRUNK:
pageTypeCount[type]++;
writer.println("-- page " + page + ": log trunk");
break;
// type 8
case Page.TYPE_STREAM_DATA:
pageTypeCount[type]++;
writer.println("-- page " + page + ": log data");
break;
default:
......@@ -844,13 +862,19 @@ public class Recover extends Tool implements DataHandler {
break;
}
}
writer.println("-- page count: " + pageCount + " empty: " + emptyPages + " free: " + free);
writeSchema(writer);
try {
dumpPageLogStream(writer, store, logFirstTrunkPage, logFirstDataPage, pageSize);
} catch (EOFException e) {
// ignore
}
writer.println("-- page count: " + pageCount + " empty: " + emptyPages + " free: " + free);
for (int i = 0; i < pageTypeCount.length; i++) {
int count = pageTypeCount[i];
if (count > 0) {
writer.println("-- page count type: " + i + " " + (100 * count / pageCount) + "% count: " + count);
}
}
writer.close();
} catch (Throwable e) {
writeError(writer, e);
......
......@@ -398,4 +398,34 @@ public class DateTimeUtils {
return value;
}
/**
* Get the number of milliseconds since 1970-01-01 in the local timezone.
*
* @param d the date
* @return the milliseconds
*/
public static long getTimeLocal(java.util.Date d) {
Calendar c = getCalendar();
synchronized (c) {
c.setTime(d);
return c.getTime().getTime() + c.get(Calendar.ZONE_OFFSET);
}
}
/**
* Convert the number of milliseconds since 1970-01-01 in the local timezone
* to GMT.
*
* @param millis the number of milliseconds in the local timezone
* @return the number of milliseconds in GMT
*/
public static long getTimeGMT(long millis) {
Date d = new Date(millis);
Calendar c = getCalendar();
synchronized (c) {
c.setTime(d);
return c.getTime().getTime() - c.get(Calendar.ZONE_OFFSET);
}
}
}
......@@ -20,6 +20,16 @@ import org.h2.util.MathUtils;
*/
public class ValueDecimal extends Value {
/**
* The value 'zero'.
*/
public static final Object ZERO = new ValueDecimal(BigDecimal.ZERO);
/**
* The value 'one'.
*/
public static final Object ONE = new ValueDecimal(BigDecimal.ONE);
/**
* The default precision for a decimal value.
*/
......@@ -36,10 +46,6 @@ public class ValueDecimal extends Value {
static final int DEFAULT_DISPLAY_SIZE = 65535;
private static final int DIVIDE_SCALE_ADD = 25;
private static final BigDecimal DEC_ZERO = new BigDecimal("0");
private static final BigDecimal DEC_ONE = new BigDecimal("1");
private static final Object ZERO = new ValueDecimal(DEC_ZERO);
private static final Object ONE = new ValueDecimal(DEC_ONE);
private final BigDecimal value;
private String valueString;
......@@ -82,7 +88,7 @@ public class ValueDecimal extends Value {
}
BigDecimal bd = value.divide(dec.value, value.scale() + DIVIDE_SCALE_ADD, BigDecimal.ROUND_HALF_DOWN);
if (bd.signum() == 0) {
bd = DEC_ZERO;
bd = BigDecimal.ZERO;
} else if (bd.scale() > 0) {
if (!bd.unscaledValue().testBit(0)) {
String s = bd.toString();
......@@ -185,9 +191,9 @@ public class ValueDecimal extends Value {
* @return the value
*/
public static ValueDecimal get(BigDecimal dec) {
if (DEC_ZERO.equals(dec)) {
if (BigDecimal.ZERO.equals(dec)) {
return (ValueDecimal) ZERO;
} else if (DEC_ONE.equals(dec)) {
} else if (BigDecimal.ONE.equals(dec)) {
return (ValueDecimal) ONE;
}
return (ValueDecimal) Value.cache(new ValueDecimal(dec));
......
......@@ -349,6 +349,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
test.pageStore = true;
test.runTests();
TestPerformance.main("-init", "-db", "1");
// Recover.execute("data", null);
System.setProperty(SysProperties.H2_PAGE_STORE, "false");
test.pageStore = false;
......
......@@ -6,10 +6,15 @@
*/
package org.h2.test.unit;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import org.h2.constant.SysProperties;
import org.h2.message.Trace;
import org.h2.store.Data;
import org.h2.store.DataHandler;
import org.h2.store.DataPage;
import org.h2.store.FileStore;
......@@ -17,11 +22,25 @@ import org.h2.test.TestBase;
import org.h2.util.SmallLRUCache;
import org.h2.util.TempFileDeleter;
import org.h2.value.Value;
import org.h2.value.ValueArray;
import org.h2.value.ValueBoolean;
import org.h2.value.ValueByte;
import org.h2.value.ValueBytes;
import org.h2.value.ValueDate;
import org.h2.value.ValueDecimal;
import org.h2.value.ValueDouble;
import org.h2.value.ValueFloat;
import org.h2.value.ValueInt;
import org.h2.value.ValueJavaObject;
import org.h2.value.ValueLong;
import org.h2.value.ValueNull;
import org.h2.value.ValueShort;
import org.h2.value.ValueString;
import org.h2.value.ValueStringFixed;
import org.h2.value.ValueStringIgnoreCase;
import org.h2.value.ValueTime;
import org.h2.value.ValueTimestamp;
import org.h2.value.ValueUuid;
/**
* Data page tests.
......@@ -38,9 +57,104 @@ public class TestDataPage extends TestBase implements DataHandler {
}
public void test() throws SQLException {
testValues();
testAll();
}
private void testValues() throws SQLException {
testValue(ValueNull.INSTANCE);
testValue(ValueBoolean.get(false));
testValue(ValueBoolean.get(true));
for (int i = 0; i < 256; i++) {
testValue(ValueByte.get((byte) i));
}
for (int i = 0; i < 256 * 256; i += 10) {
testValue(ValueShort.get((short) i));
}
for (int i = 0; i < 256 * 256; i += 10) {
testValue(ValueInt.get(i));
testValue(ValueInt.get(-i));
testValue(ValueLong.get(i));
testValue(ValueLong.get(-i));
}
testValue(ValueInt.get(Integer.MAX_VALUE));
testValue(ValueInt.get(Integer.MIN_VALUE));
for (long i = 0; i < Integer.MAX_VALUE; i += 10 + i / 4) {
testValue(ValueInt.get((int) i));
testValue(ValueInt.get((int) -i));
}
testValue(ValueLong.get(Long.MAX_VALUE));
testValue(ValueLong.get(Long.MIN_VALUE));
for (long i = 0; i >= 0; i += 10 + i / 4) {
testValue(ValueLong.get(i));
testValue(ValueLong.get(-i));
}
testValue(ValueDecimal.get(BigDecimal.ZERO));
testValue(ValueDecimal.get(BigDecimal.ONE));
testValue(ValueDecimal.get(BigDecimal.TEN));
testValue(ValueDecimal.get(BigDecimal.ONE.negate()));
testValue(ValueDecimal.get(BigDecimal.TEN.negate()));
for (long i = 0; i >= 0; i += 10 + i / 4) {
testValue(ValueDecimal.get(new BigDecimal(i)));
testValue(ValueDecimal.get(new BigDecimal(-i)));
for (int j = 0; j < 200; j += 50) {
testValue(ValueDecimal.get(new BigDecimal(i).setScale(j)));
testValue(ValueDecimal.get(new BigDecimal(i * i).setScale(j)));
}
testValue(ValueDecimal.get(new BigDecimal(i * i)));
}
testValue(ValueDate.get(new Date(System.currentTimeMillis())));
testValue(ValueDate.get(new Date(0)));
testValue(ValueTime.get(new Time(System.currentTimeMillis())));
testValue(ValueTime.get(new Time(0)));
testValue(ValueTimestamp.get(new Timestamp(System.currentTimeMillis())));
testValue(ValueTimestamp.get(new Timestamp(0)));
testValue(ValueJavaObject.getNoCopy(new byte[0]));
testValue(ValueJavaObject.getNoCopy(new byte[100]));
for (int i = 0; i < 300; i++) {
testValue(ValueBytes.getNoCopy(new byte[i]));
}
for (int i = 0; i < 65000; i += 10 + i) {
testValue(ValueBytes.getNoCopy(new byte[i]));
}
testValue(ValueUuid.getNewRandom());
for (int i = 0; i < 100; i++) {
testValue(ValueString.get(new String(new char[i])));
}
for (int i = 0; i < 65000; i += 10 + i) {
testValue(ValueString.get(new String(new char[i])));
testValue(ValueStringFixed.get(new String(new char[i])));
testValue(ValueStringIgnoreCase.get(new String(new char[i])));
}
testValue(ValueFloat.get(0f));
testValue(ValueFloat.get(1f));
testValue(ValueFloat.get(-1f));
testValue(ValueDouble.get(0));
testValue(ValueDouble.get(1));
testValue(ValueDouble.get(-1));
for (int i = 0; i < 65000; i += 10 + i) {
for (double j = 0.1; j < 65000; j += 10 + j) {
testValue(ValueFloat.get((float) (i / j)));
testValue(ValueDouble.get(i / j));
testValue(ValueFloat.get((float) -(i / j)));
testValue(ValueDouble.get(-(i / j)));
}
}
testValue(ValueArray.get(new Value[0]));
testValue(ValueArray.get(new Value[] {ValueBoolean.get(true), ValueInt.get(10)}));
}
private void testValue(Value v) throws SQLException {
Data data = Data.create(null, 1024);
data.checkCapacity((int) v.getPrecision());
data.writeValue(v);
data.reset();
Value v2 = data.readValue();
assertEquals(v.getType(), v2.getType());
assertTrue(v.compareEqual(v2));
}
private void testAll() throws SQLException {
DataPage page = DataPage.create(this, 128);
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论