More accurate cache size calculation.

h2/src/main/org/h2/engine/Database.java

@@ -1597,7 +1597,7 @@ public class Database implements DataHandler {
         }
         cacheSize = kb;
         if (pageStore != null) {
-            pageStore.getCache().setMaxSize(kb);
+            pageStore.getCache().setMaxMemory(kb);
         }
     }
 

h2/src/main/org/h2/engine/UndoLogRecord.java

@@ -230,7 +230,7 @@ public class UndoLogRecord {
         for (int i = 0; i < columnCount; i++) {
             values[i] = buff.readValue();
         }
-        row = new Row(values, 1);
+        row = new Row(values, Row.MEMORY_CALCULATE);
         row.setKey(key);
         row.setDeleted(deleted);
         row.setSessionId(sessionId);

h2/src/main/org/h2/index/PageBtree.java

@@ -70,10 +70,16 @@ public abstract class PageBtree extends Page {
      */
     protected boolean written;
 
+    /**
+     * The estimated memory used by this object.
+     */
+    protected int memoryEstimated;
+
     PageBtree(PageBtreeIndex index, int pageId, Data data) {
         this.index = index;
         this.data = data;
         setPos(pageId);
+        memoryEstimated = index.getMemoryPerPage();
     }
 
     /**
@@ -260,9 +266,12 @@ public abstract class PageBtree extends Page {
      *
      * @return number of double words (4 bytes)
      */
-    public int getMemorySize() {
-        // four times the byte array size
-        return index.getPageStore().getPageSize();
+    public int getMemory() {
+        // need to always return the same value for the same object (otherwise
+        // the cache size would change after adding and then removing the same
+        // page from the cache) but index.getMemoryPerPage() can adopt according
+        // to how much memory a row needs on average
+        return memoryEstimated;
     }
 
     public boolean canRemove() {

h2/src/main/org/h2/index/PageBtreeIndex.java

@@ -8,6 +8,7 @@ package org.h2.index;
 
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
+import org.h2.engine.Constants;
 import org.h2.engine.Session;
 import org.h2.message.DbException;
 import org.h2.result.Row;
@@ -32,6 +33,8 @@ public class PageBtreeIndex extends PageIndex {
     private RegularTable tableData;
     private boolean needRebuild;
     private long rowCount;
+    private int memoryPerPage;
+    private int memoryCount;
 
     public PageBtreeIndex(RegularTable table, int id, String indexName, IndexColumn[] columns,
             IndexType indexType, boolean create, Session session) {
@@ -424,4 +427,22 @@ public class PageBtreeIndex extends PageIndex {
         return row.getValue(columns[0].getColumnId()) != null;
     }
 
+    int getMemoryPerPage() {
+        return memoryPerPage;
+    }
+
+    /**
+     * The memory usage of a page was changed. The new value is used to adopt
+     * the average estimated memory size of a page.
+     *
+     * @param x the new memory size
+     */
+    void memoryChange(int x) {
+        if (memoryCount < Constants.MEMORY_FACTOR) {
+            memoryPerPage += (x - memoryPerPage) / ++memoryCount;
+        } else {
+            memoryPerPage += (x > memoryPerPage ? 1 : -1) + ((x - memoryPerPage) / Constants.MEMORY_FACTOR);
+        }
+    }
+
 }

h2/src/main/org/h2/index/PageBtreeLeaf.java

@@ -8,6 +8,7 @@ package org.h2.index;
 
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
+import org.h2.engine.Constants;
 import org.h2.engine.Session;
 import org.h2.message.DbException;
 import org.h2.result.SearchRow;
@@ -152,6 +153,7 @@ public class PageBtreeLeaf extends PageBtree {
         offsets = newOffsets;
         rows = newRows;
         index.getPageStore().update(this);
+        memoryChange();
         return -1;
     }
 
@@ -177,6 +179,7 @@ public class PageBtreeLeaf extends PageBtree {
         start -= OFFSET_LENGTH;
         offsets = newOffsets;
         rows = newRows;
+        memoryChange();
     }
 
     int getEntryCount() {
@@ -262,6 +265,7 @@ public class PageBtreeLeaf extends PageBtree {
             index.writeRow(data, offsets[i], rows[i], onlyPosition);
         }
         written = true;
+        memoryChange();
     }
 
     void find(PageBtreeCursor cursor, SearchRow first, boolean bigger) {
@@ -339,4 +343,17 @@ public class PageBtreeLeaf extends PageBtree {
         store.free(getPos());
     }
 
+    private void memoryChange() {
+        int memory = Constants.MEMORY_PAGE_BTREE + index.getPageStore().getPageSize();
+        if (rows != null) {
+            memory += getEntryCount() * (4 + Constants.MEMORY_POINTER);
+            for (SearchRow r : rows) {
+                if (r != null) {
+                    memory += r.getMemory();
+                }
+            }
+        }
+        index.memoryChange(memory >> 2);
+    }
+
 }

h2/src/main/org/h2/index/PageData.java

@@ -61,10 +61,16 @@ abstract class PageData extends Page {
      */
     protected boolean written;
 
+    /**
+     * The estimated memory used by this object.
+     */
+    protected int memoryEstimated;
+
     PageData(PageDataIndex index, int pageId, Data data) {
         this.index = index;
         this.data = data;
         setPos(pageId);
+        memoryEstimated = index.getMemoryPerPage();
     }
 
     /**
@@ -215,9 +221,12 @@ abstract class PageData extends Page {
      *
      * @return number of double words (4 bytes)
      */
-    public int getMemorySize() {
-        // four times the byte array size
-        return index.getPageStore().getPageSize();
+    public int getMemory() {
+        // need to always return the same value for the same object (otherwise
+        // the cache size would change after adding and then removing the same
+        // page from the cache) but index.getMemoryPerPage() can adopt according
+        // to how much memory a row needs on average
+        return memoryEstimated;
     }
 
     int getParentPageId() {
@@ -231,4 +240,6 @@ abstract class PageData extends Page {
         return true;
     }
 
+
+
 }

h2/src/main/org/h2/index/PageDataIndex.java

@@ -45,7 +45,8 @@ public class PageDataIndex extends PageIndex {
     private HashMap<Integer, Integer> sessionRowCount;
     private int mainIndexColumn = -1;
     private DbException fastDuplicateKeyException;
-    private int memorySizePerPage;
+    private int memoryPerPage;
+    private int memoryCount;
 
     public PageDataIndex(RegularTable table, int id, IndexColumn[] columns, IndexType indexType, boolean create, Session session) {
         initBaseIndex(table, id, table.getName() + "_DATA", columns, indexType);
@@ -79,10 +80,8 @@ public class PageDataIndex extends PageIndex {
         table.setRowCount(rowCount);
         // estimate the memory usage as follows:
         // the less column, the more memory is required,
-        // because the more rows fit on a page
-        memorySizePerPage = store.getPageSize();
-        int estimatedRowsPerPage =  store.getPageSize() / ((1 + columns.length) * 8);
-        memorySizePerPage += estimatedRowsPerPage * 64;
+        // because the more rows fit in a page
+        memoryPerPage = (Constants.MEMORY_PAGE_DATA + store.getPageSize()) >> 2;
     }
 
     public DbException getDuplicateKeyException() {
@@ -502,10 +501,6 @@ public class PageDataIndex extends PageIndex {
         return mainIndexColumn;
     }
 
-    int getMemorySizePerPage() {
-        return memorySizePerPage;
-    }
-
     public String toString() {
         return getName();
     }
@@ -528,4 +523,22 @@ public class PageDataIndex extends PageIndex {
         return table.getSQL() + ".tableScan";
     }
 
+    int getMemoryPerPage() {
+        return memoryPerPage;
+    }
+
+    /**
+     * The memory usage of a page was changed. The new value is used to adopt
+     * the average estimated memory size of a page.
+     *
+     * @param x the new memory size
+     */
+    void memoryChange(int x) {
+        if (memoryCount < Constants.MEMORY_FACTOR) {
+            memoryPerPage += (x - memoryPerPage) / ++memoryCount;
+        } else {
+            memoryPerPage += (x > memoryPerPage ? 1 : -1) + ((x - memoryPerPage) / Constants.MEMORY_FACTOR);
+        }
+    }
+
 }

h2/src/main/org/h2/index/PageDataLeaf.java

@@ -10,6 +10,7 @@ import java.lang.ref.SoftReference;
 import java.util.Arrays;
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
+import org.h2.engine.Constants;
 import org.h2.engine.Session;
 import org.h2.message.DbException;
 import org.h2.result.Row;
@@ -65,7 +66,7 @@ public class PageDataLeaf extends PageData {
 
     private int columnCount;
 
-    private int memorySize;
+    private int memoryData;
 
     private PageDataLeaf(PageDataIndex index, int pageId, Data data) {
         super(index, pageId, data);
@@ -204,7 +205,6 @@ public class PageDataLeaf extends PageData {
         newOffsets[x] = offset;
         newKeys[x] = row.getKey();
         newRows[x] = row;
-        memorySize += row.getMemorySize();
         offsets = newOffsets;
         keys = newKeys;
         rows = newRows;
@@ -226,7 +226,8 @@ public class PageDataLeaf extends PageData {
             this.overflowRowSize = pageSize + rowLength;
             writeData();
             // free up the space used by the row
-            rowRef = new SoftReference<Row>(rows[0]);
+            Row r = rows[0];
+            rowRef = new SoftReference<Row>(r);
             rows[0] = null;
             Data all = index.getPageStore().createData();
             all.checkCapacity(data.length());
@@ -251,6 +252,11 @@ public class PageDataLeaf extends PageData {
                 page = next;
             } while (remaining > 0);
         }
+        if (rowRef == null) {
+            memoryChange(true, row);
+        } else {
+            memoryChange(true, null);
+        }
         return -1;
     }
 
@@ -261,7 +267,7 @@ public class PageDataLeaf extends PageData {
         readAllRows();
         Row r = rows[i];
         if (r != null) {
-            memorySize += r.getMemorySize();
+            memoryChange(false, r);
         }
         entryCount--;
         if (entryCount < 0) {
@@ -336,7 +342,7 @@ public class PageDataLeaf extends PageData {
                 rowRef = new SoftReference<Row>(r);
             } else {
                 rows[at] = r;
-                memorySize += r.getMemorySize();
+                memoryChange(true, r);
             }
         }
         return r;
@@ -551,8 +557,10 @@ public class PageDataLeaf extends PageData {
         index.getPageStore().update(this);
     }
 
-    public int getMemorySize() {
-        return index.getMemorySizePerPage();
+    private void memoryChange(boolean add, Row r) {
+        int diff = r == null ? 0 : 4 + 8 + Constants.MEMORY_POINTER + r.getMemory();
+        memoryData += add ? diff : -diff;
+        index.memoryChange((Constants.MEMORY_PAGE_DATA + memoryData + index.getPageStore().getPageSize()) >> 2);
     }
 
 }

h2/src/main/org/h2/index/PageDataOverflow.java

@@ -8,6 +8,7 @@ package org.h2.index;
 
 import org.h2.constant.ErrorCode;
 import org.h2.constant.SysProperties;
+import org.h2.engine.Constants;
 import org.h2.engine.Session;
 import org.h2.message.DbException;
 import org.h2.store.Data;
@@ -198,9 +199,8 @@ public class PageDataOverflow extends Page {
      *
      * @return number of double words (4 bytes)
      */
-    public int getMemorySize() {
-        // double the byte array size
-        return store.getPageSize() >> 1;
+    public int getMemory() {
+        return (Constants.MEMORY_PAGE_DATA_OVERFLOW + store.getPageSize()) >> 2;
     }
 
     void setParentPageId(int parent) {

h2/src/main/org/h2/result/ResultTempTable.java

@@ -142,7 +142,7 @@ public class ResultTempTable implements ResultExternal {
 
     private Row convertToRow(Value[] values) {
         ValueArray data = ValueArray.get(values);
-        return new Row(new Value[]{data}, data.getMemory());
+        return new Row(new Value[]{data}, Row.MEMORY_CALCULATE);
     }
 
     private Cursor find(Row row) {

h2/src/main/org/h2/result/Row.java

@@ -6,6 +6,7 @@
  */
 package org.h2.result;
 
+import org.h2.engine.Constants;
 import org.h2.store.Data;
 import org.h2.util.StatementBuilder;
 import org.h2.value.Value;
@@ -26,7 +27,7 @@ public class Row implements SearchRow {
     public Row(Value[] data, int memory) {
         this.data = data;
         if (memory != MEMORY_CALCULATE) {
-            this.memory = 16 + memory * 4;
+            this.memory = memory;
         } else {
             this.memory = MEMORY_CALCULATE;
         }
@@ -83,13 +84,20 @@ public class Row implements SearchRow {
         return data.length;
     }
 
-    public int getMemorySize() {
+    public int getMemory() {
         if (memory != MEMORY_CALCULATE) {
             return memory;
         }
-        int m = 8;
-        for (int i = 0; data != null && i < data.length; i++) {
-            m += data[i].getMemory();
+        int m = Constants.MEMORY_ROW;
+        if (data != null) {
+            int len = data.length;
+            m += Constants.MEMORY_OBJECT + len * Constants.MEMORY_POINTER;
+            for (int i = 0; i < len; i++) {
+                Value v = data[i];
+                if (v != null) {
+                    m += v.getMemory();
+                }
+            }
         }
         return m;
     }

h2/src/main/org/h2/result/RowList.java

@@ -48,7 +48,7 @@ public class RowList {
     private void writeRow(Data buff, Row r) {
         buff.checkCapacity(1 + Data.LENGTH_INT * 8);
         buff.writeByte((byte) 1);
-        buff.writeInt(r.getMemorySize());
+        buff.writeInt(r.getMemory());
         buff.writeInt(r.getColumnCount());
         buff.writeLong(r.getKey());
         buff.writeInt(r.getVersion());
@@ -126,7 +126,7 @@ public class RowList {
      */
     public void add(Row r) {
         list.add(r);
-        memory += r.getMemorySize();
+        memory += r.getMemory() + Constants.MEMORY_POINTER;
         if (maxMemory > 0 && memory > maxMemory) {
             writeAllRows();
         }

h2/src/main/org/h2/result/SearchRow.java

@@ -65,4 +65,11 @@ public interface SearchRow {
      */
     long getKey();
 
+    /**
+     * Get the estimated memory used for this row, in bytes.
+     *
+     * @return the memory
+     */
+    int getMemory();
+
 }

h2/src/main/org/h2/result/SimpleRow.java

@@ -6,6 +6,7 @@
  */
 package org.h2.result;
 
+import org.h2.engine.Constants;
 import org.h2.util.StatementBuilder;
 import org.h2.value.Value;
 
@@ -17,6 +18,7 @@ public class SimpleRow implements SearchRow {
     private long key;
     private int version;
     private Value[] data;
+    private int memory;
 
     public SimpleRow(Value[] data) {
         this.data = data;
@@ -65,4 +67,18 @@ public class SimpleRow implements SearchRow {
         return buff.append(')').toString();
     }
 
+    public int getMemory() {
+        if (memory == 0) {
+            int len = data.length;
+            memory = Constants.MEMORY_OBJECT + len * Constants.MEMORY_POINTER;
+            for (int i = 0; i < len; i++) {
+                Value v = data[i];
+                if (v != null) {
+                    memory += v.getMemory();
+                }
+            }
+        }
+        return memory;
+    }
+
 }

h2/src/main/org/h2/result/SimpleRowValue.java

@@ -6,6 +6,7 @@
  */
 package org.h2.result;
 
+import org.h2.engine.Constants;
 import org.h2.value.Value;
 
 /**
@@ -57,4 +58,8 @@ public class SimpleRowValue implements SearchRow {
         return "( /* " + key + " */ " + (data == null ? "null" : data.getTraceSQL()) + " )";
     }
 
+    public int getMemory() {
+        return Constants.MEMORY_OBJECT + (data == null ? 0 : data.getMemory());
+    }
+
 }

h2/src/main/org/h2/store/PageFreeList.java

@@ -194,7 +194,7 @@ public class PageFreeList extends Page {
      *
      * @return number of double words (4 bytes)
      */
-    public int getMemorySize() {
+    public int getMemory() {
         return store.getPageSize() >> 2;
     }
 

h2/src/main/org/h2/store/PageStreamData.java

@@ -141,7 +141,7 @@ public class PageStreamData extends Page {
      *
      * @return number of double words (4 bytes)
      */
-    public int getMemorySize() {
+    public int getMemory() {
         return store.getPageSize() >> 2;
     }
 

h2/src/main/org/h2/store/PageStreamTrunk.java

@@ -178,7 +178,7 @@ public class PageStreamTrunk extends Page {
      *
      * @return number of double words (4 bytes)
      */
-    public int getMemorySize() {
+    public int getMemory() {
         return store.getPageSize() >> 2;
     }
 

h2/src/main/org/h2/table/MetaTable.java

@@ -891,8 +891,8 @@ public class MetaTable extends Table {
                 add(rows, "info.FILE_WRITE_TOTAL", "" + store.getWriteCountTotal());
                 add(rows, "info.FILE_WRITE", "" + store.getWriteCount());
                 add(rows, "info.FILE_READ", "" + store.getReadCount());
-                add(rows, "info.CACHE_MAX_SIZE", "" + store.getCache().getMaxSize());
-                add(rows, "info.CACHE_SIZE", "" + store.getCache().getSize());
+                add(rows, "info.CACHE_MAX_SIZE", "" + store.getCache().getMaxMemory());
+                add(rows, "info.CACHE_SIZE", "" + store.getCache().getMemory());
             }
             break;
         }

h2/src/main/org/h2/table/RegularTable.java

@@ -599,7 +599,7 @@ public class RegularTable extends TableBase {
      * @return the row
      */
     public Row createRow(Value[] data) {
-        return new Row(data, memoryPerRow);
+        return new Row(data, Row.MEMORY_CALCULATE);
     }
 
     /**

h2/src/main/org/h2/table/Table.java

@@ -492,7 +492,7 @@ public abstract class Table extends SchemaObjectBase {
     }
 
     public Row getTemplateRow() {
-        return new Row(new Value[columns.length], memoryPerRow);
+        return new Row(new Value[columns.length], Row.MEMORY_CALCULATE);
     }
 
     /**

h2/src/main/org/h2/util/Cache.java

@@ -73,20 +73,20 @@ public interface Cache {
      *
      * @param size the maximum size in KB
      */
-    void setMaxSize(int size);
+    void setMaxMemory(int size);
 
     /**
-     * Get the maximum size in KB.
+     * Get the maximum memory to be used.
      *
      * @return the maximum size in KB
      */
-    int getMaxSize();
+    int getMaxMemory();
 
     /**
      * Get the used size in KB.
      *
      * @return the current size in KB
      */
-    int getSize();
+    int getMemory();
 
 }

h2/src/main/org/h2/util/CacheHead.java

@@ -15,7 +15,7 @@ public class CacheHead extends CacheObject {
         return false;
     }
 
-    public int getMemorySize() {
+    public int getMemory() {
         return 0;
     }
 

h2/src/main/org/h2/util/CacheLRU.java

@@ -34,17 +34,17 @@ public class CacheLRU implements Cache {
     /**
      * The maximum memory, in words (4 bytes each).
      */
-    private int maxSize;
+    private int maxMemory;
 
     /**
      * The current memory used in this cache, in words (4 bytes each).
      */
-    private int sizeMemory;
+    private int memory;
 
-    private CacheLRU(CacheWriter writer, int maxKb) {
-        this.maxSize = maxKb * 1024 / 4;
+    private CacheLRU(CacheWriter writer, int maxMemoryKb) {
+        this.setMaxMemory(maxMemoryKb);
         this.writer = writer;
-        this.len = MathUtils.nextPowerOf2(maxSize / 64);
+        this.len = MathUtils.nextPowerOf2(maxMemory / 64);
         this.mask = len - 1;
         MathUtils.checkPowerOf2(len);
         clear();
@@ -82,7 +82,7 @@ public class CacheLRU implements Cache {
         values = null;
         values = new CacheObject[len];
         recordCount = 0;
-        sizeMemory = 0;
+        memory = len * Constants.MEMORY_POINTER;
     }
 
     public void put(CacheObject rec) {
@@ -97,7 +97,7 @@ public class CacheLRU implements Cache {
         rec.cacheChained = values[index];
         values[index] = rec;
         recordCount++;
-        sizeMemory += rec.getMemorySize();
+        memory += rec.getMemory();
         addToFront(rec);
         removeOldIfRequired();
     }
@@ -120,7 +120,7 @@ public class CacheLRU implements Cache {
 
     private void removeOldIfRequired() {
         // a small method, to allow inlining
-        if (sizeMemory >= maxSize) {
+        if (memory >= maxMemory) {
             removeOld();
         }
     }
@@ -128,11 +128,11 @@ public class CacheLRU implements Cache {
     private void removeOld() {
         int i = 0;
         ArrayList<CacheObject> changed = New.arrayList();
-        int mem = sizeMemory;
+        int mem = memory;
         int rc = recordCount;
         boolean flushed = false;
         CacheObject next = head.cacheNext;
-        while (mem * 4 > maxSize * 3 && rc > Constants.CACHE_MIN_RECORDS) {
+        while (mem * 4 > maxMemory * 3 && rc > Constants.CACHE_MIN_RECORDS) {
             CacheObject check = next;
             next = check.cacheNext;
             i++;
@@ -144,7 +144,7 @@ public class CacheLRU implements Cache {
                 } else {
                     // can't remove any record, because the records can not be removed
                     // hopefully this does not happen frequently, but it can happen
-                    writer.getTrace().info("Cannot remove records, cache size too small? records:" + recordCount + " memory:" + sizeMemory);
+                    writer.getTrace().info("Cannot remove records, cache size too small? records:" + recordCount + " memory:" + memory);
                     break;
                 }
             }
@@ -160,7 +160,7 @@ public class CacheLRU implements Cache {
                 continue;
             }
             rc--;
-            mem -= check.getMemorySize();
+            mem -= check.getMemory();
             if (check.isChanged()) {
                 changed.add(check);
             } else {
@@ -169,17 +169,17 @@ public class CacheLRU implements Cache {
         }
         if (changed.size() > 0) {
             Collections.sort(changed);
-            int max = maxSize;
+            int max = maxMemory;
             try {
                 // temporary disable size checking,
                 // to avoid stack overflow
-                maxSize = Integer.MAX_VALUE;
+                maxMemory = Integer.MAX_VALUE;
                 for (i = 0; i < changed.size(); i++) {
                     CacheObject rec = changed.get(i);
                     writer.writeBack(rec);
                 }
             } finally {
-                maxSize = max;
+                maxMemory = max;
             }
             for (i = 0; i < changed.size(); i++) {
                 CacheObject rec = changed.get(i);
@@ -235,7 +235,7 @@ public class CacheLRU implements Cache {
             last.cacheChained = rec.cacheChained;
         }
         recordCount--;
-        sizeMemory -= rec.getMemorySize();
+        memory -= rec.getMemory();
         removeFromLinkedList(rec);
         if (SysProperties.CHECK) {
             rec.cacheChained = null;
@@ -307,20 +307,26 @@ public class CacheLRU implements Cache {
         return list;
     }
 
-    public void setMaxSize(int maxKb) {
-        int newSize = maxKb * 1024 / 4;
-        maxSize = newSize < 0 ? 0 : newSize;
+    public void setMaxMemory(int maxKb) {
+        int newSize = MathUtils.convertLongToInt(maxKb * 1024L / 4);
+        maxMemory = newSize < 0 ? 0 : newSize;
         // can not resize, otherwise existing records are lost
         // resize(maxSize);
         removeOldIfRequired();
     }
 
-    public int getMaxSize() {
-        return maxSize * 4 / 1024;
+    public int getMaxMemory() {
+        return (int) (maxMemory * 4L / 1024);
     }
 
-    public int getSize() {
-        return sizeMemory * 4 / 1024;
+    public int getMemory() {
+        // CacheObject rec = head.cacheNext;
+        // while (rec != head) {
+            // System.out.println(rec.getMemory() + " " +
+            //        MemoryFootprint.getObjectSize(rec) + " " + rec);
+            // rec = rec.cacheNext;
+        // }
+        return (int) (memory * 4L / 1024);
     }
 
 }

h2/src/main/org/h2/util/CacheObject.java

@@ -43,11 +43,11 @@ public abstract class CacheObject implements Comparable<CacheObject> {
     public abstract boolean canRemove();
 
     /**
-     * Get the estimated memory size.
+     * Get the estimated used memory.
      *
-     * @return number of double words (4 bytes)
+     * @return number of words (one word is 4 bytes)
      */
-    public abstract int getMemorySize();
+    public abstract int getMemory();
 
     public void setPos(int pos) {
         if (SysProperties.CHECK && (cachePrevious != null || cacheNext != null || cacheChained != null)) {

h2/src/main/org/h2/util/CacheSecondLevel.java

@@ -48,12 +48,12 @@ class CacheSecondLevel implements Cache {
         return baseCache.getAllChanged();
     }
 
-    public int getMaxSize() {
-        return baseCache.getMaxSize();
+    public int getMaxMemory() {
+        return baseCache.getMaxMemory();
     }
 
-    public int getSize() {
-        return baseCache.getSize();
+    public int getMemory() {
+        return baseCache.getMemory();
     }
 
     public void put(CacheObject r) {
@@ -66,8 +66,8 @@ class CacheSecondLevel implements Cache {
         map.remove(pos);
     }
 
-    public void setMaxSize(int size) {
-        baseCache.setMaxSize(size);
+    public void setMaxMemory(int size) {
+        baseCache.setMaxMemory(size);
     }
 
     public CacheObject update(int pos, CacheObject record) {

h2/src/main/org/h2/util/Profiler.java

@@ -6,6 +6,7 @@
  */
 package org.h2.util;
 
+import java.lang.instrument.Instrumentation;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -15,6 +16,7 @@ import org.h2.engine.Constants;
  * A simple CPU profiling tool similar to java -Xrunhprof.
  */
 public class Profiler implements Runnable {
+    private static Instrumentation instrumentation;
     private static final int MAX_ELEMENTS = 1000;
 
     public int interval = 10;
@@ -29,7 +31,8 @@ public class Profiler implements Runnable {
             "java.lang.UNIXProcess.waitForProcessExit," +
             "java.lang.Object.wait," +
             "java.lang.Thread.sleep," +
-            "sun.awt.windows.WToolkit.eventLoop,"
+            "sun.awt.windows.WToolkit.eventLoop," +
+            "sun.misc.Unsafe.park,"
             , ',', true);
     private volatile boolean stop;
     private HashMap<String, Integer> counts = new HashMap<String, Integer>();
@@ -38,6 +41,25 @@ public class Profiler implements Runnable {
     private Thread thread;
     private long time;
 
+    /**
+     * This method is called when the agent is installed.
+     *
+     * @param agentArgs the agent arguments
+     * @param inst the instrumentation object
+     */
+    public static void premain(String agentArgs, Instrumentation inst) {
+        instrumentation = inst;
+    }
+
+    /**
+     * Get the instrumentation object if started as an agent.
+     *
+     * @return the instrumentation, or null
+     */
+    public static Instrumentation getInstrumentation() {
+        return instrumentation;
+    }
+
     /**
      * Start collecting profiling data.
      */

h2/src/main/org/h2/util/Utils.java

@@ -437,29 +437,23 @@ public class Utils {
      * @return the resource data
      */
     public static byte[] getResource(String name) throws IOException {
-        byte[] data;
-        if (RESOURCES.size() == 0) {
-            // TODO web: security (check what happens with files like 'lpt1.txt' on windows)
-            InputStream in = Utils.class.getResourceAsStream(name);
-            if (in == null) {
-                data = null;
-            } else {
-                data = IOUtils.readBytesAndClose(in, 0);
-            }
-        } else {
-            data = RESOURCES.get(name);
+        byte[] data = RESOURCES.get(name);
+        if (data == null) {
+            data = loadResource(name);
+            RESOURCES.put(name, data);
         }
         return data == null ? EMPTY_BYTES : data;
     }
 
-    static {
-        loadResourcesFromZip();
-    }
-
-    private static void loadResourcesFromZip() {
+    private static byte[] loadResource(String name) throws IOException {
         InputStream in = Utils.class.getResourceAsStream("data.zip");
         if (in == null) {
-            return;
+            in = Utils.class.getResourceAsStream(name);
+            if (in == null) {
+                return null;
+            } else {
+                return IOUtils.readBytesAndClose(in, 0);
+            }
         }
         ZipInputStream zipIn = new ZipInputStream(in);
         try {
@@ -472,16 +466,22 @@ public class Utils {
                 if (!entryName.startsWith("/")) {
                     entryName = "/" + entryName;
                 }
-                ByteArrayOutputStream out = new ByteArrayOutputStream();
-                IOUtils.copy(zipIn, out);
-                zipIn.closeEntry();
-                RESOURCES.put(entryName, out.toByteArray());
+                if (entryName.equals(name)) {
+                    ByteArrayOutputStream out = new ByteArrayOutputStream();
+                    IOUtils.copy(zipIn, out);
+                    zipIn.closeEntry();
+                    return out.toByteArray();
+                } else {
+                    zipIn.closeEntry();
+                }
             }
-            zipIn.close();
         } catch (IOException e) {
             // if this happens we have a real problem
             e.printStackTrace();
+        } finally {
+            zipIn.close();
         }
+        return null;
     }
 
     /**

h2/src/main/org/h2/value/DataType.java

@@ -198,132 +198,144 @@ public class DataType {
         add(Value.NULL, Types.NULL, "Null",
                 new DataType(),
                 new String[]{"NULL"},
-                1
+                // the value is always in the cache
+                0
         );
         add(Value.STRING, Types.VARCHAR, "String",
                 createString(true),
                 new String[]{"VARCHAR", "VARCHAR2", "NVARCHAR", "NVARCHAR2", "VARCHAR_CASESENSITIVE", "CHARACTER VARYING", "TID"},
-                4
+                // 24 for ValueString, 24 for String
+                48
         );
         add(Value.STRING, Types.LONGVARCHAR, "String",
                 createString(true),
                 new String[]{"LONGVARCHAR", "LONGNVARCHAR"},
-                4
+                48
         );
         add(Value.STRING_FIXED, Types.CHAR, "String",
                 createString(true),
                 new String[]{"CHAR", "CHARACTER", "NCHAR"},
-                4
+                48
         );
         add(Value.STRING_IGNORECASE, Types.VARCHAR, "String",
                 createString(false),
                 new String[]{"VARCHAR_IGNORECASE"},
-                4
+                48
         );
         add(Value.BOOLEAN, DataType.TYPE_BOOLEAN, "Boolean",
                 createDecimal(ValueBoolean.PRECISION, ValueBoolean.PRECISION, 0, ValueBoolean.DISPLAY_SIZE, false, false),
                 new String[]{"BOOLEAN", "BIT", "BOOL"},
-                1
+                // the value is always in the cache
+                0
         );
         add(Value.BYTE, Types.TINYINT, "Byte",
                 createDecimal(ValueByte.PRECISION, ValueByte.PRECISION, 0, ValueByte.DISPLAY_SIZE, false, false),
                 new String[]{"TINYINT"},
+                // the value is almost always in the cache
                 1
         );
         add(Value.SHORT, Types.SMALLINT, "Short",
                 createDecimal(ValueShort.PRECISION, ValueShort.PRECISION, 0, ValueShort.DISPLAY_SIZE, false, false),
                 new String[]{"SMALLINT", "YEAR", "INT2"},
-                5
+                // in many cases the value is in the cache
+                20
         );
         add(Value.INT, Types.INTEGER, "Int",
                 createDecimal(ValueInt.PRECISION, ValueInt.PRECISION, 0, ValueInt.DISPLAY_SIZE, false, false),
                 new String[]{"INTEGER", "INT", "MEDIUMINT", "INT4", "SIGNED"},
-                5
+                // in many cases the value is in the cache
+                20
         );
         add(Value.LONG, Types.BIGINT, "Long",
                 createDecimal(ValueLong.PRECISION, ValueLong.PRECISION, 0, ValueLong.DISPLAY_SIZE, false, false),
                 new String[]{"BIGINT", "INT8", "LONG"},
-                5
+                24
         );
         add(Value.LONG, Types.BIGINT, "Long",
                 createDecimal(ValueLong.PRECISION, ValueLong.PRECISION, 0, ValueLong.DISPLAY_SIZE, false, true),
                 new String[]{"IDENTITY", "SERIAL"},
-                5
+                24
         );
         add(Value.DECIMAL, Types.DECIMAL, "BigDecimal",
                 createDecimal(Integer.MAX_VALUE, ValueDecimal.DEFAULT_PRECISION, ValueDecimal.DEFAULT_SCALE, ValueDecimal.DEFAULT_DISPLAY_SIZE, true, false),
                 new String[]{"DECIMAL", "DEC"},
-                17
+                // 40 for ValueDecimal,
+                64
         );
         add(Value.DECIMAL, Types.NUMERIC, "BigDecimal",
                 createDecimal(Integer.MAX_VALUE, ValueDecimal.DEFAULT_PRECISION, ValueDecimal.DEFAULT_SCALE, ValueDecimal.DEFAULT_DISPLAY_SIZE, true, false),
                 new String[]{"NUMERIC", "NUMBER"},
-                17
+                64
         );
         add(Value.FLOAT, Types.REAL, "Float",
                 createDecimal(ValueFloat.PRECISION, ValueFloat.PRECISION, 0, ValueFloat.DISPLAY_SIZE, false, false),
                 new String[] {"REAL", "FLOAT4"},
-                5
+                24
         );
         add(Value.DOUBLE, Types.DOUBLE, "Double",
                 createDecimal(ValueDouble.PRECISION, ValueDouble.PRECISION, 0, ValueDouble.DISPLAY_SIZE, false, false),
                 new String[] { "DOUBLE", "DOUBLE PRECISION" },
-                4
+                24
         );
         add(Value.DOUBLE, Types.FLOAT, "Double",
                 createDecimal(ValueDouble.PRECISION, ValueDouble.PRECISION, 0, ValueDouble.DISPLAY_SIZE, false, false),
                 new String[] {"FLOAT", "FLOAT8" },
-                4
+                24
         );
         add(Value.TIME, Types.TIME, "Time",
                 createDate(ValueTime.PRECISION, "TIME", 0, ValueTime.DISPLAY_SIZE),
                 new String[]{"TIME"},
-                10
+                // 24 for ValueTime, 32 for java.sql.Time
+                56
         );
         add(Value.DATE, Types.DATE, "Date",
                 createDate(ValueDate.PRECISION, "DATE", 0, ValueDate.DISPLAY_SIZE),
                 new String[]{"DATE"},
-                10
+                // 24 for ValueDate, 32 for java.sql.Data
+                56
         );
         add(Value.TIMESTAMP, Types.TIMESTAMP, "Timestamp",
                 createDate(ValueTimestamp.PRECISION, "TIMESTAMP", ValueTimestamp.DEFAULT_SCALE, ValueTimestamp.DISPLAY_SIZE),
                 new String[]{"TIMESTAMP", "DATETIME", "SMALLDATETIME"},
-                12
+                // 24 for ValueTimestamp, 32 for java.sql.Timestamp
+                56
         );
         add(Value.BYTES, Types.VARBINARY, "Bytes",
                 createString(false),
                 new String[]{"VARBINARY"},
-                8
+                32
         );
         add(Value.BYTES, Types.BINARY, "Bytes",
                 createString(false),
                 new String[]{"BINARY", "RAW", "BYTEA", "LONG RAW"},
-                8
+                32
         );
         add(Value.BYTES, Types.LONGVARBINARY, "Bytes",
                 createString(false),
                 new String[]{"LONGVARBINARY"},
-                8
+                32
         );
         add(Value.UUID, Types.BINARY, "Bytes",
                 createString(false),
                 new String[]{"UUID"},
-                8
+                32
         );
         add(Value.JAVA_OBJECT, Types.OTHER, "Object",
                 createString(false),
                 new String[]{"OTHER", "OBJECT", "JAVA_OBJECT"},
-                8
+                24
         );
         add(Value.BLOB, Types.BLOB, "Blob",
                 createLob(),
                 new String[]{"BLOB", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", "IMAGE", "OID"},
-                10
+                // 80 for ValueLob, 24 for String
+                104
         );
         add(Value.CLOB, Types.CLOB, "Clob",
                 createLob(),
                 new String[]{"CLOB", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT", "NTEXT", "NCLOB"},
-                10
+                // 80 for ValueLob, 24 for String
+                104
         );
         DataType dataType = new DataType();
         dataType.prefix = "(";
@@ -331,13 +343,13 @@ public class DataType {
         add(Value.ARRAY, Types.ARRAY, "Array",
                 dataType,
                 new String[]{"ARRAY"},
-                10
+                32
         );
         dataType = new DataType();
         add(Value.RESULT_SET, 0, "ResultSet",
                 dataType,
                 new String[]{"RESULT_SET"},
-                20
+                400
         );
         for (int i = 0; i < TYPES_BY_VALUE_TYPE.size(); i++) {
             DataType dt = TYPES_BY_VALUE_TYPE.get(i);

h2/src/main/org/h2/value/Value.java

@@ -189,7 +189,7 @@ public abstract class Value {
      * @return the memory used in bytes
      */
     public int getMemory() {
-        return DataType.getDataType(getType()).memory * 4;
+        return DataType.getDataType(getType()).memory;
     }
 
     /**
@@ -352,6 +352,13 @@ public abstract class Value {
         return v;
     }
 
+    /**
+     * Clear the value cache. Used for testing.
+     */
+    public static void clearCache() {
+        softCache = null;
+    }
+
     public Boolean getBoolean() {
         return ((ValueBoolean) convertTo(Value.BOOLEAN)).getBoolean();
     }

h2/src/main/org/h2/value/ValueArray.java

@@ -7,6 +7,7 @@
 package org.h2.value;
 
 import java.sql.PreparedStatement;
+import org.h2.engine.Constants;
 import org.h2.util.MathUtils;
 import org.h2.util.StatementBuilder;
 
@@ -142,9 +143,9 @@ public class ValueArray extends Value {
     }
 
     public int getMemory() {
-        int memory = 0;
+        int memory = 32;
         for (Value v : values) {
-            memory += v.getMemory();
+            memory += v.getMemory() + Constants.MEMORY_POINTER;
         }
         return memory;
     }

h2/src/main/org/h2/value/ValueBytes.java

@@ -110,7 +110,7 @@ public class ValueBytes extends Value {
     }
 
     public int getMemory() {
-        return value.length + 4;
+        return value.length + 24;
     }
 
     public boolean equals(Object other) {

h2/src/main/org/h2/value/ValueDecimal.java

@@ -211,7 +211,7 @@ public class ValueDecimal extends Value {
     }
 
     public int getMemory() {
-        return MathUtils.precision(value) * 3 + 120;
+        return MathUtils.precision(value) + 120;
     }
 
 }

h2/src/main/org/h2/value/ValueLob.java

@@ -760,9 +760,9 @@ public class ValueLob extends Value {
 
     public int getMemory() {
         if (small != null) {
-            return small.length + 64;
+            return small.length + 104;
         }
-        return 128;
+        return 140;
     }
 
     /**

h2/src/main/org/h2/value/ValueLobDb.java

@@ -334,9 +334,9 @@ public class ValueLobDb extends Value implements Value.ValueClob, Value.ValueBlo
 
     public int getMemory() {
         if (small != null) {
-            return small.length + 64;
+            return small.length + 104;
         }
-        return 128;
+        return 140;
     }
 
     /**

h2/src/main/org/h2/value/ValueString.java

@@ -64,7 +64,7 @@ public class ValueString extends Value {
     }
 
     public int getMemory() {
-        return value.length() * 2 + 30;
+        return value.length() * 2 + 48;
     }
 
     public Value convertPrecision(long precision) {

h2/src/test/org/h2/test/unit/TestCache.java

@@ -8,6 +8,7 @@ package org.h2.test.unit;
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Random;
@@ -18,6 +19,9 @@ import org.h2.util.Cache;
 import org.h2.util.CacheLRU;
 import org.h2.util.CacheObject;
 import org.h2.util.CacheWriter;
+import org.h2.util.StringUtils;
+import org.h2.util.Utils;
+import org.h2.value.Value;
 
 /**
  * Tests the cache.
@@ -32,24 +36,80 @@ public class TestCache extends TestBase implements CacheWriter {
      * @param a ignored
      */
     public static void main(String... a) throws Exception {
-        TestBase.createCaller().init().test();
+        TestBase test = TestBase.createCaller().init();
+//        test.config.traceTest = true;
+        test.test();
     }
 
     public void test() throws SQLException {
-        // testCache(false);
-        testCache(true);
+        testMemoryUsage();
+        testCache();
         testCacheDb(false);
         testCacheDb(true);
     }
 
-    private void testCache(boolean lru) {
+    private void testMemoryUsage() throws SQLException {
+        if (!config.traceTest) {
+            return;
+        }
+        if (config.memory) {
+            return;
+        }
+        deleteDb("cache");
+        Connection conn;
+        Statement stat;
+        ResultSet rs;
+        conn = getConnection("cache;CACHE_SIZE=16384");
+        stat = conn.createStatement();
+        // test DataOverflow
+        stat.execute("create table test(id int primary key, data varchar)");
+        stat.execute("set max_memory_undo 10000");
+        conn.close();
+        stat = null;
+        conn = null;
+        long before = getRealMemory();
+
+        conn = getConnection("cache;CACHE_SIZE=16384;DB_CLOSE_ON_EXIT=FALSE");
+        stat = conn.createStatement();
+
+        //  -XX:+HeapDumpOnOutOfMemoryError
+
+        stat.execute("insert into test select x, random_uuid() || space(1) from system_range(1, 10000)");
+
+        // stat.execute("create index idx_test_n on test(data)");
+        // stat.execute("select data from test where data >= ''");
+
+        rs = stat.executeQuery("select value from information_schema.settings where name = 'info.CACHE_SIZE'");
+        rs.next();
+        int calculated = rs.getInt(1);
+        rs = null;
+        long afterInsert = getRealMemory();
+
+        conn.close();
+        stat = null;
+        conn = null;
+        rs = null;
+        long afterClose = getRealMemory();
+        trace("Used memory: " + (afterInsert - afterClose) + " calculated cache size: " + calculated);
+        trace("Before: " + before + " after: " + afterInsert + " after closing: " + afterClose);
+    }
+
+    private int getRealMemory() {
+        StringUtils.clearCache();
+        Value.clearCache();
+        eatMemory(100);
+        freeMemory();
+        System.gc();
+        return Utils.getMemoryUsed();
+    }
+
+    private void testCache() {
         out = "";
-        Cache c = CacheLRU.getCache(this, lru ? "LRU" : "TQ", 16);
+        Cache c = CacheLRU.getCache(this, "LRU", 16);
         for (int i = 0; i < 20; i++) {
             c.put(new Obj(i));
         }
-        assertEquals(lru ? "0 1 2 3 " : "4 5 6 ", out);
-
+        assertEquals("0 1 2 3 ", out);
     }
 
     /**
@@ -61,7 +121,7 @@ public class TestCache extends TestBase implements CacheWriter {
             setPos(pos);
         }
 
-        public int getMemorySize() {
+        public int getMemory() {
             return 1024;
         }
 

h2/src/test/org/h2/test/unit/TestValueMemory.java

@@ -14,10 +14,13 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.IdentityHashMap;
 import java.util.Random;
+import org.h2.engine.Constants;
 import org.h2.store.DataHandler;
 import org.h2.store.FileStore;
 import org.h2.store.LobStorage;
 import org.h2.test.TestBase;
+import org.h2.test.utils.MemoryFootprint;
+import org.h2.tools.SimpleResultSet;
 import org.h2.util.SmallLRUCache;
 import org.h2.util.TempFileDeleter;
 import org.h2.util.Utils;
@@ -34,6 +37,7 @@ import org.h2.value.ValueInt;
 import org.h2.value.ValueJavaObject;
 import org.h2.value.ValueLong;
 import org.h2.value.ValueNull;
+import org.h2.value.ValueResultSet;
 import org.h2.value.ValueShort;
 import org.h2.value.ValueString;
 import org.h2.value.ValueStringFixed;
@@ -58,12 +62,21 @@ public class TestValueMemory extends TestBase implements DataHandler {
      * @param a ignored
      */
     public static void main(String... a) throws Exception {
-        TestBase.createCaller().init().test();
+        // run using -javaagent:ext/h2-1.2.139.jar
+        TestBase test = TestBase.createCaller().init();
+        test.config.traceTest = true;
+        test.test();
     }
 
     public void test() throws SQLException {
         testCompare();
         for (int i = 0; i < Value.TYPE_COUNT; i++) {
+            Value v = create(i);
+            String s = "type: " + v.getType() + " calculated: " + v.getMemory() + " real: " + MemoryFootprint.getObjectSize(v) + " " + v.getClass().getName() + ": " + v.toString();
+            trace(s);
+        }
+        for (int i = 0; i < Value.TYPE_COUNT; i++) {
+            assertEquals(i, create(i).getType());
             testType(i);
         }
     }
@@ -83,7 +96,7 @@ public class TestValueMemory extends TestBase implements DataHandler {
         long memory = 0;
         for (int i = 0; memory < 1000000; i++) {
             Value v = create(type);
-            memory += v.getMemory();
+            memory += v.getMemory() + Constants.MEMORY_POINTER;
             list.add(v);
         }
         Object[] array = list.toArray();
@@ -99,8 +112,11 @@ public class TestValueMemory extends TestBase implements DataHandler {
         System.gc();
         long used = Utils.getMemoryUsed() - first;
         memory /= 1024;
+        if (config.traceTest) {
+            trace("Type: " + type + " Used memory: " + used + " calculated: " + memory + " length: " + array.length + " size: " + size);
+        }
         if (used > memory * 3) {
-            fail("Type: " + type + " Used memory: " + used + " calculated: " + memory + " " + array.length + " size: " + size);
+            fail("Type: " + type + " Used memory: " + used + " calculated: " + memory + " length: " + array.length + " size: " + size);
         }
     }
     private Value create(int type) throws SQLException {
@@ -155,8 +171,7 @@ public class TestValueMemory extends TestBase implements DataHandler {
             return ValueArray.get(list);
         }
         case Value.RESULT_SET:
-            // not supported currently
-            return ValueNull.INSTANCE;
+            return ValueResultSet.get(new SimpleResultSet());
         case Value.JAVA_OBJECT:
             return ValueJavaObject.getNoCopy(randomBytes(random.nextInt(100)));
         case Value.UUID: