#630 Integer overflow in CacheLRU can cause unrestricted cache growth

h2/src/docsrc/html/changelog.html

@@ -21,6 +21,8 @@ Change Log
 
 <h2>Next Version (unreleased)</h2>
 <ul>
+<li>Issue #630: Integer overflow in CacheLRU can cause unrestricted cache growth
+</li>
 <li>Issue #497: Fix TO_DATE in cases of 'inline' text. E.g. the "T" and "Z" in to_date('2017-04-21T00:00:00Z', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')
 </li>
 <li>Fix bug in MySQL/ORACLE-syntax silently corrupting the modified column in cases of setting the 'NULL'- or 'NOT NULL'-constraint. E.g. alter table T modify C NULL;

h2/src/main/org/h2/util/CacheLRU.java

@@ -40,18 +40,22 @@ public class CacheLRU implements Cache {
     /**
      * The maximum memory, in words (4 bytes each).
      */
-    private int maxMemory;
+    private long maxMemory;
 
     /**
      * The current memory used in this cache, in words (4 bytes each).
      */
-    private int memory;
+    private long memory;
 
     CacheLRU(CacheWriter writer, int maxMemoryKb, boolean fifo) {
         this.writer = writer;
         this.fifo = fifo;
         this.setMaxMemory(maxMemoryKb);
-        this.len = MathUtils.nextPowerOf2(maxMemory / 64);
+        long tmpLen = MathUtils.nextPowerOf2(maxMemory / 64);
+        if (tmpLen > Integer.MAX_VALUE) {
+            throw new IllegalStateException("do not support this much cache memory: " + maxMemoryKb + "kb");
+        }
+        this.len = (int) tmpLen;
         this.mask = len - 1;
         clear();
     }
@@ -92,7 +96,7 @@ public class CacheLRU implements Cache {
         values = null;
         values = new CacheObject[len];
         recordCount = 0;
-        memory = len * Constants.MEMORY_POINTER;
+        memory = len * (long)Constants.MEMORY_POINTER;
     }
 
     @Override
@@ -145,7 +149,7 @@ public class CacheLRU implements Cache {
     private void removeOld() {
         int i = 0;
         ArrayList<CacheObject> changed = New.arrayList();
-        int mem = memory;
+        long mem = memory;
         int rc = recordCount;
         boolean flushed = false;
         CacheObject next = head.cacheNext;
@@ -204,12 +208,12 @@ public class CacheLRU implements Cache {
                 writer.flushLog();
             }
             Collections.sort(changed);
-            int max = maxMemory;
+            long max = maxMemory;
             int size = changed.size();
             try {
                 // temporary disable size checking,
                 // to avoid stack overflow
-                maxMemory = Integer.MAX_VALUE;
+                maxMemory = Long.MAX_VALUE;
                 for (i = 0; i < size; i++) {
                     CacheObject rec = changed.get(i);
                     writer.writeBack(rec);
@@ -352,7 +356,7 @@ public class CacheLRU implements Cache {
 
     @Override
     public void setMaxMemory(int maxKb) {
-        int newSize = MathUtils.convertLongToInt(maxKb * 1024L / 4);
+        long newSize = maxKb * 1024L / 4;
         maxMemory = newSize < 0 ? 0 : newSize;
         // can not resize, otherwise existing records are lost
         // resize(maxSize);

h2/src/main/org/h2/util/MathUtils.java

@@ -227,6 +227,21 @@ public class MathUtils {
         return (int) i;
     }
 
+    /**
+     * Get the value that is equal or higher than this value, and that is a
+     * power of two.
+     *
+     * @param x the original value
+     * @return the next power of two value
+     */
+    public static long nextPowerOf2(long x) {
+        long i = 1;
+        while (i < x && i < (Long.MAX_VALUE / 2)) {
+            i += i;
+        }
+        return i;
+    }
+
     /**
      * Convert a long value to an int value. Values larger than the biggest int
      * value is converted to the biggest int value, and values smaller than the