Skip to content

H
h2database
提交 202e687f authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

More accurate cache size calculation.

上级 1cab2114
隐藏空白字符变更
内嵌 并排
正在显示 包含 390 行增加 和 152 行删除
h2/src/main/org/h2/engine/Database.java
浏览文件 @ 202e687f
......@@ -1597,7 +1597,7 @@ public class Database implements DataHandler {
}
cacheSize = kb;
if (pageStore != null) {
pageStore.getCache().setMaxSize(kb);
pageStore.getCache().setMaxMemory(kb);
}
}
......
h2/src/main/org/h2/engine/UndoLogRecord.java
浏览文件 @ 202e687f
......@@ -230,7 +230,7 @@ public class UndoLogRecord {
for (int i = 0; i < columnCount; i++) {
values[i] = buff.readValue();
}
row = new Row(values, 1);
row = new Row(values, Row.MEMORY_CALCULATE);
row.setKey(key);
row.setDeleted(deleted);
row.setSessionId(sessionId);
......
h2/src/main/org/h2/index/PageBtree.java
浏览文件 @ 202e687f
......@@ -70,10 +70,16 @@ public abstract class PageBtree extends Page {
*/
protected boolean written;
/**
* The estimated memory used by this object.
*/
protected int memoryEstimated;
PageBtree(PageBtreeIndex index, int pageId, Data data) {
this.index = index;
this.data = data;
setPos(pageId);
memoryEstimated = index.getMemoryPerPage();
}
/**
......@@ -260,9 +266,12 @@ public abstract class PageBtree extends Page {
*
* @return number of double words (4 bytes)
*/
public int getMemorySize() {
// four times the byte array size
return index.getPageStore().getPageSize();
public int getMemory() {
// need to always return the same value for the same object (otherwise
// the cache size would change after adding and then removing the same
// page from the cache) but index.getMemoryPerPage() can adopt according
// to how much memory a row needs on average
return memoryEstimated;
}
public boolean canRemove() {
......
h2/src/main/org/h2/index/PageBtreeIndex.java
浏览文件 @ 202e687f
......@@ -8,6 +8,7 @@ package org.h2.index;
import org.h2.constant.ErrorCode;
import org.h2.constant.SysProperties;
import org.h2.engine.Constants;
import org.h2.engine.Session;
import org.h2.message.DbException;
import org.h2.result.Row;
......@@ -32,6 +33,8 @@ public class PageBtreeIndex extends PageIndex {
private RegularTable tableData;
private boolean needRebuild;
private long rowCount;
private int memoryPerPage;
private int memoryCount;
public PageBtreeIndex(RegularTable table, int id, String indexName, IndexColumn[] columns,
IndexType indexType, boolean create, Session session) {
......@@ -424,4 +427,22 @@ public class PageBtreeIndex extends PageIndex {
return row.getValue(columns[0].getColumnId()) != null;
}
int getMemoryPerPage() {
return memoryPerPage;
}
/**
* The memory usage of a page was changed. The new value is used to adopt
* the average estimated memory size of a page.
*
* @param x the new memory size
*/
void memoryChange(int x) {
if (memoryCount < Constants.MEMORY_FACTOR) {
memoryPerPage += (x - memoryPerPage) / ++memoryCount;
} else {
memoryPerPage += (x > memoryPerPage ? 1 : -1) + ((x - memoryPerPage) / Constants.MEMORY_FACTOR);
}
}
}
h2/src/main/org/h2/index/PageBtreeLeaf.java
浏览文件 @ 202e687f
......@@ -8,6 +8,7 @@ package org.h2.index;
import org.h2.constant.ErrorCode;
import org.h2.constant.SysProperties;
import org.h2.engine.Constants;
import org.h2.engine.Session;
import org.h2.message.DbException;
import org.h2.result.SearchRow;
......@@ -152,6 +153,7 @@ public class PageBtreeLeaf extends PageBtree {
offsets = newOffsets;
rows = newRows;
index.getPageStore().update(this);
memoryChange();
return -1;
}
......@@ -177,6 +179,7 @@ public class PageBtreeLeaf extends PageBtree {
start -= OFFSET_LENGTH;
offsets = newOffsets;
rows = newRows;
memoryChange();
}
int getEntryCount() {
......@@ -262,6 +265,7 @@ public class PageBtreeLeaf extends PageBtree {
index.writeRow(data, offsets[i], rows[i], onlyPosition);
}
written = true;
memoryChange();
}
void find(PageBtreeCursor cursor, SearchRow first, boolean bigger) {
......@@ -339,4 +343,17 @@ public class PageBtreeLeaf extends PageBtree {
store.free(getPos());
}
private void memoryChange() {
int memory = Constants.MEMORY_PAGE_BTREE + index.getPageStore().getPageSize();
if (rows != null) {
memory += getEntryCount() * (4 + Constants.MEMORY_POINTER);
for (SearchRow r : rows) {
if (r != null) {
memory += r.getMemory();
}
}
}
index.memoryChange(memory >> 2);
}
}
h2/src/main/org/h2/index/PageData.java
浏览文件 @ 202e687f
......@@ -61,10 +61,16 @@ abstract class PageData extends Page {
*/
protected boolean written;
/**
* The estimated memory used by this object.
*/
protected int memoryEstimated;
PageData(PageDataIndex index, int pageId, Data data) {
this.index = index;
this.data = data;
setPos(pageId);
memoryEstimated = index.getMemoryPerPage();
}
/**
......@@ -215,9 +221,12 @@ abstract class PageData extends Page {
*
* @return number of double words (4 bytes)
*/
public int getMemorySize() {
// four times the byte array size
return index.getPageStore().getPageSize();
public int getMemory() {
// need to always return the same value for the same object (otherwise
// the cache size would change after adding and then removing the same
// page from the cache) but index.getMemoryPerPage() can adopt according
// to how much memory a row needs on average
return memoryEstimated;
}
int getParentPageId() {
......@@ -231,4 +240,6 @@ abstract class PageData extends Page {
return true;
}
}
h2/src/main/org/h2/index/PageDataIndex.java
浏览文件 @ 202e687f
......@@ -45,7 +45,8 @@ public class PageDataIndex extends PageIndex {
private HashMap<Integer, Integer> sessionRowCount;
private int mainIndexColumn = -1;
private DbException fastDuplicateKeyException;
private int memorySizePerPage;
private int memoryPerPage;
private int memoryCount;
public PageDataIndex(RegularTable table, int id, IndexColumn[] columns, IndexType indexType, boolean create, Session session) {
initBaseIndex(table, id, table.getName() + "_DATA", columns, indexType);
......@@ -79,10 +80,8 @@ public class PageDataIndex extends PageIndex {
table.setRowCount(rowCount);
// estimate the memory usage as follows:
// the less column, the more memory is required,
// because the more rows fit on a page
memorySizePerPage = store.getPageSize();
int estimatedRowsPerPage = store.getPageSize() / ((1 + columns.length) * 8);
memorySizePerPage += estimatedRowsPerPage * 64;
// because the more rows fit in a page
memoryPerPage = (Constants.MEMORY_PAGE_DATA + store.getPageSize()) >> 2;
}
public DbException getDuplicateKeyException() {
......@@ -502,10 +501,6 @@ public class PageDataIndex extends PageIndex {
return mainIndexColumn;
}
int getMemorySizePerPage() {
return memorySizePerPage;
}
public String toString() {
return getName();
}
......@@ -528,4 +523,22 @@ public class PageDataIndex extends PageIndex {
return table.getSQL() + ".tableScan";
}
int getMemoryPerPage() {
return memoryPerPage;
}
/**
* The memory usage of a page was changed. The new value is used to adopt
* the average estimated memory size of a page.
*
* @param x the new memory size
*/
void memoryChange(int x) {
if (memoryCount < Constants.MEMORY_FACTOR) {
memoryPerPage += (x - memoryPerPage) / ++memoryCount;
} else {
memoryPerPage += (x > memoryPerPage ? 1 : -1) + ((x - memoryPerPage) / Constants.MEMORY_FACTOR);
}
}
}
h2/src/main/org/h2/index/PageDataLeaf.java
浏览文件 @ 202e687f
......@@ -10,6 +10,7 @@ import java.lang.ref.SoftReference;
import java.util.Arrays;
import org.h2.constant.ErrorCode;
import org.h2.constant.SysProperties;
import org.h2.engine.Constants;
import org.h2.engine.Session;
import org.h2.message.DbException;
import org.h2.result.Row;
......@@ -65,7 +66,7 @@ public class PageDataLeaf extends PageData {
private int columnCount;
private int memorySize;
private int memoryData;
private PageDataLeaf(PageDataIndex index, int pageId, Data data) {
super(index, pageId, data);
......@@ -204,7 +205,6 @@ public class PageDataLeaf extends PageData {
newOffsets[x] = offset;
newKeys[x] = row.getKey();
newRows[x] = row;
memorySize += row.getMemorySize();
offsets = newOffsets;
keys = newKeys;
rows = newRows;
......@@ -226,7 +226,8 @@ public class PageDataLeaf extends PageData {
this.overflowRowSize = pageSize + rowLength;
writeData();
// free up the space used by the row
rowRef = new SoftReference<Row>(rows[0]);
Row r = rows[0];
rowRef = new SoftReference<Row>(r);
rows[0] = null;
Data all = index.getPageStore().createData();
all.checkCapacity(data.length());
......@@ -251,6 +252,11 @@ public class PageDataLeaf extends PageData {
page = next;
} while (remaining > 0);
}
if (rowRef == null) {
memoryChange(true, row);
} else {
memoryChange(true, null);
}
return -1;
}
......@@ -261,7 +267,7 @@ public class PageDataLeaf extends PageData {
readAllRows();
Row r = rows[i];
if (r != null) {
memorySize += r.getMemorySize();
memoryChange(false, r);
}
entryCount--;
if (entryCount < 0) {
......@@ -336,7 +342,7 @@ public class PageDataLeaf extends PageData {
rowRef = new SoftReference<Row>(r);
} else {
rows[at] = r;
memorySize += r.getMemorySize();
memoryChange(true, r);
}
}
return r;
......@@ -551,8 +557,10 @@ public class PageDataLeaf extends PageData {
index.getPageStore().update(this);
}
public int getMemorySize() {
return index.getMemorySizePerPage();
private void memoryChange(boolean add, Row r) {
int diff = r == null ? 0 : 4 + 8 + Constants.MEMORY_POINTER + r.getMemory();
memoryData += add ? diff : -diff;
index.memoryChange((Constants.MEMORY_PAGE_DATA + memoryData + index.getPageStore().getPageSize()) >> 2);
}
}
h2/src/main/org/h2/index/PageDataOverflow.java
浏览文件 @ 202e687f
......@@ -8,6 +8,7 @@ package org.h2.index;
import org.h2.constant.ErrorCode;
import org.h2.constant.SysProperties;
import org.h2.engine.Constants;
import org.h2.engine.Session;
import org.h2.message.DbException;
import org.h2.store.Data;
......@@ -198,9 +199,8 @@ public class PageDataOverflow extends Page {
*
* @return number of double words (4 bytes)
*/
public int getMemorySize() {
// double the byte array size
return store.getPageSize() >> 1;
public int getMemory() {
return (Constants.MEMORY_PAGE_DATA_OVERFLOW + store.getPageSize()) >> 2;
}
void setParentPageId(int parent) {
......
h2/src/main/org/h2/result/ResultTempTable.java
浏览文件 @ 202e687f
......@@ -142,7 +142,7 @@ public class ResultTempTable implements ResultExternal {
private Row convertToRow(Value[] values) {
ValueArray data = ValueArray.get(values);
return new Row(new Value[]{data}, data.getMemory());
return new Row(new Value[]{data}, Row.MEMORY_CALCULATE);
}
private Cursor find(Row row) {
......
h2/src/main/org/h2/result/Row.java
浏览文件 @ 202e687f
......@@ -6,6 +6,7 @@
*/
package org.h2.result;
import org.h2.engine.Constants;
import org.h2.store.Data;
import org.h2.util.StatementBuilder;
import org.h2.value.Value;
......@@ -26,7 +27,7 @@ public class Row implements SearchRow {
public Row(Value[] data, int memory) {
this.data = data;
if (memory != MEMORY_CALCULATE) {
this.memory = 16 + memory * 4;
this.memory = memory;
} else {
this.memory = MEMORY_CALCULATE;
}
......@@ -83,13 +84,20 @@ public class Row implements SearchRow {
return data.length;
}
public int getMemorySize() {
public int getMemory() {
if (memory != MEMORY_CALCULATE) {
return memory;
}
int m = 8;
for (int i = 0; data != null && i < data.length; i++) {
m += data[i].getMemory();
int m = Constants.MEMORY_ROW;
if (data != null) {
int len = data.length;
m += Constants.MEMORY_OBJECT + len * Constants.MEMORY_POINTER;
for (int i = 0; i < len; i++) {
Value v = data[i];
if (v != null) {
m += v.getMemory();
}
}
}
return m;
}
......
h2/src/main/org/h2/result/RowList.java
浏览文件 @ 202e687f
......@@ -48,7 +48,7 @@ public class RowList {
private void writeRow(Data buff, Row r) {
buff.checkCapacity(1 + Data.LENGTH_INT * 8);
buff.writeByte((byte) 1);
buff.writeInt(r.getMemorySize());
buff.writeInt(r.getMemory());
buff.writeInt(r.getColumnCount());
buff.writeLong(r.getKey());
buff.writeInt(r.getVersion());
......@@ -126,7 +126,7 @@ public class RowList {
*/
public void add(Row r) {
list.add(r);
memory += r.getMemorySize();
memory += r.getMemory() + Constants.MEMORY_POINTER;
if (maxMemory > 0 && memory > maxMemory) {
writeAllRows();
}
......
h2/src/main/org/h2/result/SearchRow.java
浏览文件 @ 202e687f
......@@ -65,4 +65,11 @@ public interface SearchRow {
*/
long getKey();
/**
* Get the estimated memory used for this row, in bytes.
*
* @return the memory
*/
int getMemory();
}
h2/src/main/org/h2/result/SimpleRow.java
浏览文件 @ 202e687f
......@@ -6,6 +6,7 @@
*/
package org.h2.result;
import org.h2.engine.Constants;
import org.h2.util.StatementBuilder;
import org.h2.value.Value;
......@@ -17,6 +18,7 @@ public class SimpleRow implements SearchRow {
private long key;
private int version;
private Value[] data;
private int memory;
public SimpleRow(Value[] data) {
this.data = data;
......@@ -65,4 +67,18 @@ public class SimpleRow implements SearchRow {
return buff.append(')').toString();
}
public int getMemory() {
if (memory == 0) {
int len = data.length;
memory = Constants.MEMORY_OBJECT + len * Constants.MEMORY_POINTER;
for (int i = 0; i < len; i++) {
Value v = data[i];
if (v != null) {
memory += v.getMemory();
}
}
}
return memory;
}
}
h2/src/main/org/h2/result/SimpleRowValue.java
浏览文件 @ 202e687f
......@@ -6,6 +6,7 @@
*/
package org.h2.result;
import org.h2.engine.Constants;
import org.h2.value.Value;
/**
......@@ -57,4 +58,8 @@ public class SimpleRowValue implements SearchRow {
return "( /* " + key + " */ " + (data == null ? "null" : data.getTraceSQL()) + " )";
}
public int getMemory() {
return Constants.MEMORY_OBJECT + (data == null ? 0 : data.getMemory());
}
}
h2/src/main/org/h2/store/PageFreeList.java
浏览文件 @ 202e687f
......@@ -194,7 +194,7 @@ public class PageFreeList extends Page {
*
* @return number of double words (4 bytes)
*/
public int getMemorySize() {
public int getMemory() {
return store.getPageSize() >> 2;
}
......
h2/src/main/org/h2/store/PageStreamData.java
浏览文件 @ 202e687f
......@@ -141,7 +141,7 @@ public class PageStreamData extends Page {
*
* @return number of double words (4 bytes)
*/
public int getMemorySize() {
public int getMemory() {
return store.getPageSize() >> 2;
}
......
h2/src/main/org/h2/store/PageStreamTrunk.java
浏览文件 @ 202e687f
......@@ -178,7 +178,7 @@ public class PageStreamTrunk extends Page {
*
* @return number of double words (4 bytes)
*/
public int getMemorySize() {
public int getMemory() {
return store.getPageSize() >> 2;
}
......
h2/src/main/org/h2/table/MetaTable.java
浏览文件 @ 202e687f
......@@ -891,8 +891,8 @@ public class MetaTable extends Table {
add(rows, "info.FILE_WRITE_TOTAL", "" + store.getWriteCountTotal());
add(rows, "info.FILE_WRITE", "" + store.getWriteCount());
add(rows, "info.FILE_READ", "" + store.getReadCount());
add(rows, "info.CACHE_MAX_SIZE", "" + store.getCache().getMaxSize());
add(rows, "info.CACHE_SIZE", "" + store.getCache().getSize());
add(rows, "info.CACHE_MAX_SIZE", "" + store.getCache().getMaxMemory());
add(rows, "info.CACHE_SIZE", "" + store.getCache().getMemory());
}
break;
}
......
h2/src/main/org/h2/table/RegularTable.java
浏览文件 @ 202e687f
......@@ -599,7 +599,7 @@ public class RegularTable extends TableBase {
* @return the row
*/
public Row createRow(Value[] data) {
return new Row(data, memoryPerRow);
return new Row(data, Row.MEMORY_CALCULATE);
}
/**
......
h2/src/main/org/h2/table/Table.java
浏览文件 @ 202e687f
......@@ -492,7 +492,7 @@ public abstract class Table extends SchemaObjectBase {
}
public Row getTemplateRow() {
return new Row(new Value[columns.length], memoryPerRow);
return new Row(new Value[columns.length], Row.MEMORY_CALCULATE);
}
/**
......
h2/src/main/org/h2/util/Cache.java
浏览文件 @ 202e687f
......@@ -73,20 +73,20 @@ public interface Cache {
*
* @param size the maximum size in KB
*/
void setMaxSize(int size);
void setMaxMemory(int size);
/**
* Get the maximum size in KB.
* Get the maximum memory to be used.
*
* @return the maximum size in KB
*/
int getMaxSize();
int getMaxMemory();
/**
* Get the used size in KB.
*
* @return the current size in KB
*/
int getSize();
int getMemory();
}
h2/src/main/org/h2/util/CacheHead.java
浏览文件 @ 202e687f
......@@ -15,7 +15,7 @@ public class CacheHead extends CacheObject {
return false;
}
public int getMemorySize() {
public int getMemory() {
return 0;
}
......
h2/src/main/org/h2/util/CacheLRU.java
浏览文件 @ 202e687f
......@@ -34,17 +34,17 @@ public class CacheLRU implements Cache {
/**
* The maximum memory, in words (4 bytes each).
*/
private int maxSize;
private int maxMemory;
/**
* The current memory used in this cache, in words (4 bytes each).
*/
private int sizeMemory;
private int memory;
private CacheLRU(CacheWriter writer, int maxKb) {
this.maxSize = maxKb * 1024 / 4;
private CacheLRU(CacheWriter writer, int maxMemoryKb) {
this.setMaxMemory(maxMemoryKb);
this.writer = writer;
this.len = MathUtils.nextPowerOf2(maxSize / 64);
this.len = MathUtils.nextPowerOf2(maxMemory / 64);
this.mask = len - 1;
MathUtils.checkPowerOf2(len);
clear();
......@@ -82,7 +82,7 @@ public class CacheLRU implements Cache {
values = null;
values = new CacheObject[len];
recordCount = 0;
sizeMemory = 0;
memory = len * Constants.MEMORY_POINTER;
}
public void put(CacheObject rec) {
......@@ -97,7 +97,7 @@ public class CacheLRU implements Cache {
rec.cacheChained = values[index];
values[index] = rec;
recordCount++;
sizeMemory += rec.getMemorySize();
memory += rec.getMemory();
addToFront(rec);
removeOldIfRequired();
}
......@@ -120,7 +120,7 @@ public class CacheLRU implements Cache {
private void removeOldIfRequired() {
// a small method, to allow inlining
if (sizeMemory >= maxSize) {
if (memory >= maxMemory) {
removeOld();
}
}
......@@ -128,11 +128,11 @@ public class CacheLRU implements Cache {
private void removeOld() {
int i = 0;
ArrayList<CacheObject> changed = New.arrayList();
int mem = sizeMemory;
int mem = memory;
int rc = recordCount;
boolean flushed = false;
CacheObject next = head.cacheNext;
while (mem * 4 > maxSize * 3 && rc > Constants.CACHE_MIN_RECORDS) {
while (mem * 4 > maxMemory * 3 && rc > Constants.CACHE_MIN_RECORDS) {
CacheObject check = next;
next = check.cacheNext;
i++;
......@@ -144,7 +144,7 @@ public class CacheLRU implements Cache {
} else {
// can't remove any record, because the records can not be removed
// hopefully this does not happen frequently, but it can happen
writer.getTrace().info("Cannot remove records, cache size too small? records:" + recordCount + " memory:" + sizeMemory);
writer.getTrace().info("Cannot remove records, cache size too small? records:" + recordCount + " memory:" + memory);
break;
}
}
......@@ -160,7 +160,7 @@ public class CacheLRU implements Cache {
continue;
}
rc--;
mem -= check.getMemorySize();
mem -= check.getMemory();
if (check.isChanged()) {
changed.add(check);
} else {
......@@ -169,17 +169,17 @@ public class CacheLRU implements Cache {
}
if (changed.size() > 0) {
Collections.sort(changed);
int max = maxSize;
int max = maxMemory;
try {
// temporary disable size checking,
// to avoid stack overflow
maxSize = Integer.MAX_VALUE;
maxMemory = Integer.MAX_VALUE;
for (i = 0; i < changed.size(); i++) {
CacheObject rec = changed.get(i);
writer.writeBack(rec);
}
} finally {
maxSize = max;
maxMemory = max;
}
for (i = 0; i < changed.size(); i++) {
CacheObject rec = changed.get(i);
......@@ -235,7 +235,7 @@ public class CacheLRU implements Cache {
last.cacheChained = rec.cacheChained;
}
recordCount--;
sizeMemory -= rec.getMemorySize();
memory -= rec.getMemory();
removeFromLinkedList(rec);
if (SysProperties.CHECK) {
rec.cacheChained = null;
......@@ -307,20 +307,26 @@ public class CacheLRU implements Cache {
return list;
}
public void setMaxSize(int maxKb) {
int newSize = maxKb * 1024 / 4;
maxSize = newSize < 0 ? 0 : newSize;
public void setMaxMemory(int maxKb) {
int newSize = MathUtils.convertLongToInt(maxKb * 1024L / 4);
maxMemory = newSize < 0 ? 0 : newSize;
// can not resize, otherwise existing records are lost
// resize(maxSize);
removeOldIfRequired();
}
public int getMaxSize() {
return maxSize * 4 / 1024;
public int getMaxMemory() {
return (int) (maxMemory * 4L / 1024);
}
public int getSize() {
return sizeMemory * 4 / 1024;
public int getMemory() {
// CacheObject rec = head.cacheNext;
// while (rec != head) {
// System.out.println(rec.getMemory() + " " +
// MemoryFootprint.getObjectSize(rec) + " " + rec);
// rec = rec.cacheNext;
// }
return (int) (memory * 4L / 1024);
}
}
......
h2/src/main/org/h2/util/CacheObject.java
浏览文件 @ 202e687f
......@@ -43,11 +43,11 @@ public abstract class CacheObject implements Comparable<CacheObject> {
public abstract boolean canRemove();
/**
* Get the estimated memory size.
* Get the estimated used memory.
*
* @return number of double words (4 bytes)
* @return number of words (one word is 4 bytes)
*/
public abstract int getMemorySize();
public abstract int getMemory();
public void setPos(int pos) {
if (SysProperties.CHECK && (cachePrevious != null || cacheNext != null || cacheChained != null)) {
......
h2/src/main/org/h2/util/CacheSecondLevel.java
浏览文件 @ 202e687f
......@@ -48,12 +48,12 @@ class CacheSecondLevel implements Cache {
return baseCache.getAllChanged();
}
public int getMaxSize() {
return baseCache.getMaxSize();
public int getMaxMemory() {
return baseCache.getMaxMemory();
}
public int getSize() {
return baseCache.getSize();
public int getMemory() {
return baseCache.getMemory();
}
public void put(CacheObject r) {
......@@ -66,8 +66,8 @@ class CacheSecondLevel implements Cache {
map.remove(pos);
}
public void setMaxSize(int size) {
baseCache.setMaxSize(size);
public void setMaxMemory(int size) {
baseCache.setMaxMemory(size);
}
public CacheObject update(int pos, CacheObject record) {
......
h2/src/main/org/h2/util/Profiler.java
浏览文件 @ 202e687f
......@@ -6,6 +6,7 @@
*/
package org.h2.util;
import java.lang.instrument.Instrumentation;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
......@@ -15,6 +16,7 @@ import org.h2.engine.Constants;
* A simple CPU profiling tool similar to java -Xrunhprof.
*/
public class Profiler implements Runnable {
private static Instrumentation instrumentation;
private static final int MAX_ELEMENTS = 1000;
public int interval = 10;
......@@ -29,7 +31,8 @@ public class Profiler implements Runnable {
"java.lang.UNIXProcess.waitForProcessExit," +
"java.lang.Object.wait," +
"java.lang.Thread.sleep," +
"sun.awt.windows.WToolkit.eventLoop,"
"sun.awt.windows.WToolkit.eventLoop," +
"sun.misc.Unsafe.park,"
, ',', true);
private volatile boolean stop;
private HashMap<String, Integer> counts = new HashMap<String, Integer>();
......@@ -38,6 +41,25 @@ public class Profiler implements Runnable {
private Thread thread;
private long time;
/**
* This method is called when the agent is installed.
*
* @param agentArgs the agent arguments
* @param inst the instrumentation object
*/
public static void premain(String agentArgs, Instrumentation inst) {
instrumentation = inst;
}
/**
* Get the instrumentation object if started as an agent.
*
* @return the instrumentation, or null
*/
public static Instrumentation getInstrumentation() {
return instrumentation;
}
/**
* Start collecting profiling data.
*/
......
h2/src/main/org/h2/util/Utils.java
浏览文件 @ 202e687f
......@@ -437,29 +437,23 @@ public class Utils {
* @return the resource data
*/
public static byte[] getResource(String name) throws IOException {
byte[] data;
if (RESOURCES.size() == 0) {
// TODO web: security (check what happens with files like 'lpt1.txt' on windows)
InputStream in = Utils.class.getResourceAsStream(name);
if (in == null) {
data = null;
} else {
data = IOUtils.readBytesAndClose(in, 0);
}
} else {
data = RESOURCES.get(name);
byte[] data = RESOURCES.get(name);
if (data == null) {
data = loadResource(name);
RESOURCES.put(name, data);
}
return data == null ? EMPTY_BYTES : data;
}
static {
loadResourcesFromZip();
}
private static void loadResourcesFromZip() {
private static byte[] loadResource(String name) throws IOException {
InputStream in = Utils.class.getResourceAsStream("data.zip");
if (in == null) {
return;
in = Utils.class.getResourceAsStream(name);
if (in == null) {
return null;
} else {
return IOUtils.readBytesAndClose(in, 0);
}
}
ZipInputStream zipIn = new ZipInputStream(in);
try {
......@@ -472,16 +466,22 @@ public class Utils {
if (!entryName.startsWith("/")) {
entryName = "/" + entryName;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copy(zipIn, out);
zipIn.closeEntry();
RESOURCES.put(entryName, out.toByteArray());
if (entryName.equals(name)) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copy(zipIn, out);
zipIn.closeEntry();
return out.toByteArray();
} else {
zipIn.closeEntry();
}
}
zipIn.close();
} catch (IOException e) {
// if this happens we have a real problem
e.printStackTrace();
} finally {
zipIn.close();
}
return null;
}
/**
......
h2/src/main/org/h2/value/DataType.java
浏览文件 @ 202e687f
......@@ -198,132 +198,144 @@ public class DataType {
add(Value.NULL, Types.NULL, "Null",
new DataType(),
new String[]{"NULL"},
1
// the value is always in the cache
0
);
add(Value.STRING, Types.VARCHAR, "String",
createString(true),
new String[]{"VARCHAR", "VARCHAR2", "NVARCHAR", "NVARCHAR2", "VARCHAR_CASESENSITIVE", "CHARACTER VARYING", "TID"},
4
// 24 for ValueString, 24 for String
48
);
add(Value.STRING, Types.LONGVARCHAR, "String",
createString(true),
new String[]{"LONGVARCHAR", "LONGNVARCHAR"},
4
48
);
add(Value.STRING_FIXED, Types.CHAR, "String",
createString(true),
new String[]{"CHAR", "CHARACTER", "NCHAR"},
4
48
);
add(Value.STRING_IGNORECASE, Types.VARCHAR, "String",
createString(false),
new String[]{"VARCHAR_IGNORECASE"},
4
48
);
add(Value.BOOLEAN, DataType.TYPE_BOOLEAN, "Boolean",
createDecimal(ValueBoolean.PRECISION, ValueBoolean.PRECISION, 0, ValueBoolean.DISPLAY_SIZE, false, false),
new String[]{"BOOLEAN", "BIT", "BOOL"},
1
// the value is always in the cache
0
);
add(Value.BYTE, Types.TINYINT, "Byte",
createDecimal(ValueByte.PRECISION, ValueByte.PRECISION, 0, ValueByte.DISPLAY_SIZE, false, false),
new String[]{"TINYINT"},
// the value is almost always in the cache
1
);
add(Value.SHORT, Types.SMALLINT, "Short",
createDecimal(ValueShort.PRECISION, ValueShort.PRECISION, 0, ValueShort.DISPLAY_SIZE, false, false),
new String[]{"SMALLINT", "YEAR", "INT2"},
5
// in many cases the value is in the cache
20
);
add(Value.INT, Types.INTEGER, "Int",
createDecimal(ValueInt.PRECISION, ValueInt.PRECISION, 0, ValueInt.DISPLAY_SIZE, false, false),
new String[]{"INTEGER", "INT", "MEDIUMINT", "INT4", "SIGNED"},
5
// in many cases the value is in the cache
20
);
add(Value.LONG, Types.BIGINT, "Long",
createDecimal(ValueLong.PRECISION, ValueLong.PRECISION, 0, ValueLong.DISPLAY_SIZE, false, false),
new String[]{"BIGINT", "INT8", "LONG"},
5
24
);
add(Value.LONG, Types.BIGINT, "Long",
createDecimal(ValueLong.PRECISION, ValueLong.PRECISION, 0, ValueLong.DISPLAY_SIZE, false, true),
new String[]{"IDENTITY", "SERIAL"},
5
24
);
add(Value.DECIMAL, Types.DECIMAL, "BigDecimal",
createDecimal(Integer.MAX_VALUE, ValueDecimal.DEFAULT_PRECISION, ValueDecimal.DEFAULT_SCALE, ValueDecimal.DEFAULT_DISPLAY_SIZE, true, false),
new String[]{"DECIMAL", "DEC"},
17
// 40 for ValueDecimal,
64
);
add(Value.DECIMAL, Types.NUMERIC, "BigDecimal",
createDecimal(Integer.MAX_VALUE, ValueDecimal.DEFAULT_PRECISION, ValueDecimal.DEFAULT_SCALE, ValueDecimal.DEFAULT_DISPLAY_SIZE, true, false),
new String[]{"NUMERIC", "NUMBER"},
17
64
);
add(Value.FLOAT, Types.REAL, "Float",
createDecimal(ValueFloat.PRECISION, ValueFloat.PRECISION, 0, ValueFloat.DISPLAY_SIZE, false, false),
new String[] {"REAL", "FLOAT4"},
5
24
);
add(Value.DOUBLE, Types.DOUBLE, "Double",
createDecimal(ValueDouble.PRECISION, ValueDouble.PRECISION, 0, ValueDouble.DISPLAY_SIZE, false, false),
new String[] { "DOUBLE", "DOUBLE PRECISION" },
4
24
);
add(Value.DOUBLE, Types.FLOAT, "Double",
createDecimal(ValueDouble.PRECISION, ValueDouble.PRECISION, 0, ValueDouble.DISPLAY_SIZE, false, false),
new String[] {"FLOAT", "FLOAT8" },
4
24
);
add(Value.TIME, Types.TIME, "Time",
createDate(ValueTime.PRECISION, "TIME", 0, ValueTime.DISPLAY_SIZE),
new String[]{"TIME"},
10
// 24 for ValueTime, 32 for java.sql.Time
56
);
add(Value.DATE, Types.DATE, "Date",
createDate(ValueDate.PRECISION, "DATE", 0, ValueDate.DISPLAY_SIZE),
new String[]{"DATE"},
10
// 24 for ValueDate, 32 for java.sql.Data
56
);
add(Value.TIMESTAMP, Types.TIMESTAMP, "Timestamp",
createDate(ValueTimestamp.PRECISION, "TIMESTAMP", ValueTimestamp.DEFAULT_SCALE, ValueTimestamp.DISPLAY_SIZE),
new String[]{"TIMESTAMP", "DATETIME", "SMALLDATETIME"},
12
// 24 for ValueTimestamp, 32 for java.sql.Timestamp
56
);
add(Value.BYTES, Types.VARBINARY, "Bytes",
createString(false),
new String[]{"VARBINARY"},
8
32
);
add(Value.BYTES, Types.BINARY, "Bytes",
createString(false),
new String[]{"BINARY", "RAW", "BYTEA", "LONG RAW"},
8
32
);
add(Value.BYTES, Types.LONGVARBINARY, "Bytes",
createString(false),
new String[]{"LONGVARBINARY"},
8
32
);
add(Value.UUID, Types.BINARY, "Bytes",
createString(false),
new String[]{"UUID"},
8
32
);
add(Value.JAVA_OBJECT, Types.OTHER, "Object",
createString(false),
new String[]{"OTHER", "OBJECT", "JAVA_OBJECT"},
8
24
);
add(Value.BLOB, Types.BLOB, "Blob",
createLob(),
new String[]{"BLOB", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", "IMAGE", "OID"},
10
// 80 for ValueLob, 24 for String
104
);
add(Value.CLOB, Types.CLOB, "Clob",
createLob(),
new String[]{"CLOB", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT", "NTEXT", "NCLOB"},
10
// 80 for ValueLob, 24 for String
104
);
DataType dataType = new DataType();
dataType.prefix = "(";
......@@ -331,13 +343,13 @@ public class DataType {
add(Value.ARRAY, Types.ARRAY, "Array",
dataType,
new String[]{"ARRAY"},
10
32
);
dataType = new DataType();
add(Value.RESULT_SET, 0, "ResultSet",
dataType,
new String[]{"RESULT_SET"},
20
400
);
for (int i = 0; i < TYPES_BY_VALUE_TYPE.size(); i++) {
DataType dt = TYPES_BY_VALUE_TYPE.get(i);
......
h2/src/main/org/h2/value/Value.java
浏览文件 @ 202e687f
......@@ -189,7 +189,7 @@ public abstract class Value {
* @return the memory used in bytes
*/
public int getMemory() {
return DataType.getDataType(getType()).memory * 4;
return DataType.getDataType(getType()).memory;
}
/**
......@@ -352,6 +352,13 @@ public abstract class Value {
return v;
}
/**
* Clear the value cache. Used for testing.
*/
public static void clearCache() {
softCache = null;
}
public Boolean getBoolean() {
return ((ValueBoolean) convertTo(Value.BOOLEAN)).getBoolean();
}
......
h2/src/main/org/h2/value/ValueArray.java
浏览文件 @ 202e687f
......@@ -7,6 +7,7 @@
package org.h2.value;
import java.sql.PreparedStatement;
import org.h2.engine.Constants;
import org.h2.util.MathUtils;
import org.h2.util.StatementBuilder;
......@@ -142,9 +143,9 @@ public class ValueArray extends Value {
}
public int getMemory() {
int memory = 0;
int memory = 32;
for (Value v : values) {
memory += v.getMemory();
memory += v.getMemory() + Constants.MEMORY_POINTER;
}
return memory;
}
......
h2/src/main/org/h2/value/ValueBytes.java
浏览文件 @ 202e687f
......@@ -110,7 +110,7 @@ public class ValueBytes extends Value {
}
public int getMemory() {
return value.length + 4;
return value.length + 24;
}
public boolean equals(Object other) {
......
h2/src/main/org/h2/value/ValueDecimal.java
浏览文件 @ 202e687f
......@@ -211,7 +211,7 @@ public class ValueDecimal extends Value {
}
public int getMemory() {
return MathUtils.precision(value) * 3 + 120;
return MathUtils.precision(value) + 120;
}
}
h2/src/main/org/h2/value/ValueLob.java
浏览文件 @ 202e687f
......@@ -760,9 +760,9 @@ public class ValueLob extends Value {
public int getMemory() {
if (small != null) {
return small.length + 64;
return small.length + 104;
}
return 128;
return 140;
}
/**
......
h2/src/main/org/h2/value/ValueLobDb.java
浏览文件 @ 202e687f
......@@ -334,9 +334,9 @@ public class ValueLobDb extends Value implements Value.ValueClob, Value.ValueBlo
public int getMemory() {
if (small != null) {
return small.length + 64;
return small.length + 104;
}
return 128;
return 140;
}
/**
......
h2/src/main/org/h2/value/ValueString.java
浏览文件 @ 202e687f
......@@ -64,7 +64,7 @@ public class ValueString extends Value {
}
public int getMemory() {
return value.length() * 2 + 30;
return value.length() * 2 + 48;
}
public Value convertPrecision(long precision) {
......
h2/src/test/org/h2/test/unit/TestCache.java
浏览文件 @ 202e687f
......@@ -8,6 +8,7 @@ package org.h2.test.unit;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Random;
......@@ -18,6 +19,9 @@ import org.h2.util.Cache;
import org.h2.util.CacheLRU;
import org.h2.util.CacheObject;
import org.h2.util.CacheWriter;
import org.h2.util.StringUtils;
import org.h2.util.Utils;
import org.h2.value.Value;
/**
* Tests the cache.
......@@ -32,24 +36,80 @@ public class TestCache extends TestBase implements CacheWriter {
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase.createCaller().init().test();
TestBase test = TestBase.createCaller().init();
// test.config.traceTest = true;
test.test();
}
public void test() throws SQLException {
// testCache(false);
testCache(true);
testMemoryUsage();
testCache();
testCacheDb(false);
testCacheDb(true);
}
private void testCache(boolean lru) {
private void testMemoryUsage() throws SQLException {
if (!config.traceTest) {
return;
}
if (config.memory) {
return;
}
deleteDb("cache");
Connection conn;
Statement stat;
ResultSet rs;
conn = getConnection("cache;CACHE_SIZE=16384");
stat = conn.createStatement();
// test DataOverflow
stat.execute("create table test(id int primary key, data varchar)");
stat.execute("set max_memory_undo 10000");
conn.close();
stat = null;
conn = null;
long before = getRealMemory();
conn = getConnection("cache;CACHE_SIZE=16384;DB_CLOSE_ON_EXIT=FALSE");
stat = conn.createStatement();
// -XX:+HeapDumpOnOutOfMemoryError
stat.execute("insert into test select x, random_uuid() || space(1) from system_range(1, 10000)");
// stat.execute("create index idx_test_n on test(data)");
// stat.execute("select data from test where data >= ''");
rs = stat.executeQuery("select value from information_schema.settings where name = 'info.CACHE_SIZE'");
rs.next();
int calculated = rs.getInt(1);
rs = null;
long afterInsert = getRealMemory();
conn.close();
stat = null;
conn = null;
rs = null;
long afterClose = getRealMemory();
trace("Used memory: " + (afterInsert - afterClose) + " calculated cache size: " + calculated);
trace("Before: " + before + " after: " + afterInsert + " after closing: " + afterClose);
}
private int getRealMemory() {
StringUtils.clearCache();
Value.clearCache();
eatMemory(100);
freeMemory();
System.gc();
return Utils.getMemoryUsed();
}
private void testCache() {
out = "";
Cache c = CacheLRU.getCache(this, lru ? "LRU" : "TQ", 16);
Cache c = CacheLRU.getCache(this, "LRU", 16);
for (int i = 0; i < 20; i++) {
c.put(new Obj(i));
}
assertEquals(lru ? "0 1 2 3 " : "4 5 6 ", out);
assertEquals("0 1 2 3 ", out);
}
/**
......@@ -61,7 +121,7 @@ public class TestCache extends TestBase implements CacheWriter {
setPos(pos);
}
public int getMemorySize() {
public int getMemory() {
return 1024;
}
......
h2/src/test/org/h2/test/unit/TestValueMemory.java
浏览文件 @ 202e687f
......@@ -14,10 +14,13 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.IdentityHashMap;
import java.util.Random;
import org.h2.engine.Constants;
import org.h2.store.DataHandler;
import org.h2.store.FileStore;
import org.h2.store.LobStorage;
import org.h2.test.TestBase;
import org.h2.test.utils.MemoryFootprint;
import org.h2.tools.SimpleResultSet;
import org.h2.util.SmallLRUCache;
import org.h2.util.TempFileDeleter;
import org.h2.util.Utils;
......@@ -34,6 +37,7 @@ import org.h2.value.ValueInt;
import org.h2.value.ValueJavaObject;
import org.h2.value.ValueLong;
import org.h2.value.ValueNull;
import org.h2.value.ValueResultSet;
import org.h2.value.ValueShort;
import org.h2.value.ValueString;
import org.h2.value.ValueStringFixed;
......@@ -58,12 +62,21 @@ public class TestValueMemory extends TestBase implements DataHandler {
* @param a ignored
*/
public static void main(String... a) throws Exception {
TestBase.createCaller().init().test();
// run using -javaagent:ext/h2-1.2.139.jar
TestBase test = TestBase.createCaller().init();
test.config.traceTest = true;
test.test();
}
public void test() throws SQLException {
testCompare();
for (int i = 0; i < Value.TYPE_COUNT; i++) {
Value v = create(i);
String s = "type: " + v.getType() + " calculated: " + v.getMemory() + " real: " + MemoryFootprint.getObjectSize(v) + " " + v.getClass().getName() + ": " + v.toString();
trace(s);
}
for (int i = 0; i < Value.TYPE_COUNT; i++) {
assertEquals(i, create(i).getType());
testType(i);
}
}
......@@ -83,7 +96,7 @@ public class TestValueMemory extends TestBase implements DataHandler {
long memory = 0;
for (int i = 0; memory < 1000000; i++) {
Value v = create(type);
memory += v.getMemory();
memory += v.getMemory() + Constants.MEMORY_POINTER;
list.add(v);
}
Object[] array = list.toArray();
......@@ -99,8 +112,11 @@ public class TestValueMemory extends TestBase implements DataHandler {
System.gc();
long used = Utils.getMemoryUsed() - first;
memory /= 1024;
if (config.traceTest) {
trace("Type: " + type + " Used memory: " + used + " calculated: " + memory + " length: " + array.length + " size: " + size);
}
if (used > memory * 3) {
fail("Type: " + type + " Used memory: " + used + " calculated: " + memory + " " + array.length + " size: " + size);
fail("Type: " + type + " Used memory: " + used + " calculated: " + memory + " length: " + array.length + " size: " + size);
}
}
private Value create(int type) throws SQLException {
......@@ -155,8 +171,7 @@ public class TestValueMemory extends TestBase implements DataHandler {
return ValueArray.get(list);
}
case Value.RESULT_SET:
// not supported currently
return ValueNull.INSTANCE;
return ValueResultSet.get(new SimpleResultSet());
case Value.JAVA_OBJECT:
return ValueJavaObject.getNoCopy(randomBytes(random.nextInt(100)));
case Value.UUID:
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论