speed up opening large databases if data was deleted

h2/src/docsrc/html/changelog.html

@@ -17,7 +17,8 @@ Change Log
 
 <h2>Next Version (unreleased)</h2>
 <ul>
-<li>RUNSCRIPT could throw a NullPointerException if the script name was an expression.
+<li>Databases larger than 1 GB was sometimes very slow if a lot of data was deleted previously. Fixed.
+</li><li>RUNSCRIPT could throw a NullPointerException if the script name was an expression.
 </li><li>Improved compatibility. New compatibility modes for Oracle and Derby. 
     New compatibility flag uniqueIndexNullDistinct to only allow one row with 'NULL' in a unique 
     index. This flag is enabled for Derby, Oracle, MSSQLServer, and HSQLDB.

h2/src/main/org/h2/constant/ErrorCode.java

@@ -1490,6 +1490,12 @@ public class ErrorCode {
      */
     public static final int CANNOT_DROP_2 = 90107;
 
+    /**
+     * The error with code <code>90108</code> is thrown when
+     * not enough memory was available to read a value from the disk.
+     */
+    public static final int OUT_OF_MEMORY_1 = 90108;
+
     /**
      * The error with code <code>90109</code> is thrown when
      * trying to run a query against an invalid view.
@@ -1806,7 +1812,7 @@ public class ErrorCode {
      */
     public static final int CAN_ONLY_ASSIGN_TO_VARIABLE_1 = 90137;
 
-    // next is 90108, 90138
+    // next is 90138
     
     private ErrorCode() {
         // utility class

h2/src/main/org/h2/res/_messages_de.properties

@@ -129,6 +129,7 @@
 90105=Fehler beim Aufruf eine benutzerdefinierten Funktion
 90106=Kann {0} nicht zur\u00FCcksetzen per TRUNCATE
 90107=Kann {0} nicht l\u00F6schen weil {1} davon abh\u00E4ngt
+90108=Nicht genug Hauptspeicher. Anzahl Bytes\: {0}
 90109=View {0} ist ung\u00FCltig\: {1}
 90110={0} ausserhalb des Bereichts
 90111=Fehler beim Zugriff auf eine verkn\u00FCpfte Tabelle mit SQL Befehl {0}, Grund\: {1}

h2/src/main/org/h2/res/_messages_en.properties

@@ -129,6 +129,7 @@
 90105=Exception calling user-defined function
 90106=Cannot truncate {0}
 90107=Cannot drop {0} because {1} depends on it
+90108=Out of memory. Size\: {0}
 90109=View {0} is invalid\: {1}
 90110={0} out of range
 90111=Error accessing linked table with SQL statement {0}, cause\: {1}

h2/src/main/org/h2/res/_messages_ja.properties

@@ -129,6 +129,7 @@
 90105=\u30E6\u30FC\u30B6\u5B9A\u7FA9\u95A2\u6570\u3092\u5B9F\u884C\u4E2D\u306B\u4F8B\u5916\u304C\u767A\u751F\u3057\u307E\u3057\u305F
 90106={0} \u3092\u7A7A\u306B\u3067\u304D\u307E\u305B\u3093
 90107={1} \u304C\u4F9D\u5B58\u3057\u3066\u3044\u308B\u305F\u3081\u3001{0} \u3092\u30C9\u30ED\u30C3\u30D7\u3059\u308B\u3053\u3068\u306F\u3067\u304D\u307E\u305B\u3093
+90108=\#Out of memory. Size\: {0}
 90109=\u30D3\u30E5\u30FC {0} \u304C\u7121\u52B9\u3067\u3059\: {1}
 90110={0} \u306F\u7BC4\u56F2\u5916\u3067\u3059
 90111=SQL\u30B9\u30C6\u30FC\u30C8\u30E1\u30F3\u30C8 {0} \u306B\u3088\u308B\u7D50\u5408\u30C6\u30FC\u30D6\u30EB\u30A2\u30AF\u30BB\u30B9\u30A8\u30E9\u30FC

h2/src/main/org/h2/res/_messages_pl.properties

@@ -129,6 +129,7 @@
 90105=Wyjatek wywoluje funkcje uzytkownika
 90106=Nie mozna obciac {0}
 90107=Nie mozna skasowac {0} poniewaz zalezy od {1}
+90108=\#Out of memory. Size\: {0}
 90109=Widok {0} jest nieprawidlowy
 90110={0} poza zakresem
 90111=\#Error accessing linked table with SQL statement {0}, cause\: {1}

h2/src/main/org/h2/res/_messages_pt_br.properties

@@ -129,6 +129,7 @@
 90105=Exce\u00E7\u00E3o na chamada da fun\u00E7\u00E3o definida pelo usu\u00E1rio
 90106=N\u00E3o pode fazer o truncate {0}
 90107=N\u00E3o pode apagar {0} por que depende de {1}
+90108=\#Out of memory. Size\: {0}
 90109=Vista {0} \u00E9 inv\u00E1lida\: {1}
 90110={0} out of range
 90111=Erro ao acessar a tabela lincada com a instru\u00E7\u00E3o SQL {0}, causa\: {1}

h2/src/main/org/h2/store/DataPage.java

@@ -545,13 +545,13 @@ public abstract class DataPage {
         }
         case Value.JAVA_OBJECT: {
             int len = readInt();
-            byte[] b = new byte[len];
+            byte[] b = createByteArray(len);
             read(b, 0, len);
             return ValueJavaObject.getNoCopy(b);
         }
         case Value.BYTES: {
             int len = readInt();
-            byte[] b = new byte[len];
+            byte[] b = createByteArray(len);
             read(b, 0, len);
             return ValueBytes.getNoCopy(b);
         }
@@ -571,7 +571,7 @@ public abstract class DataPage {
         case Value.CLOB: {
             int smallLen = readInt();
             if (smallLen >= 0) {
-                byte[] small = new byte[smallLen];
+                byte[] small = createByteArray(smallLen);
                 read(small, 0, smallLen);
                 return ValueLob.createSmallLob(dataType, small);
             }
@@ -604,6 +604,16 @@ public abstract class DataPage {
             throw Message.getInternalError("type=" + dataType);
         }
     }
+    
+    private byte[] createByteArray(int len) {
+        try {
+            return new byte[len];
+        } catch (OutOfMemoryError e) {
+            Error e2 = new OutOfMemoryError("Requested memory: " + len);
+            e2.initCause(e);
+            throw e2;
+        }
+    }
 
     /**
      * Fill up the buffer with empty space and an (initially empty) checksum

h2/src/main/org/h2/store/DiskFile.java

@@ -176,11 +176,28 @@ public class DiskFile implements CacheWriter {
     }
 
     private void freeUnusedPages() throws SQLException {
+        // first, store the unused pages and current owner in a temporary list
+        IntArray freePages = new IntArray();
+        HashSet owners = new HashSet();
         for (int i = 0; i < pageOwners.size(); i++) {
-            if (pageOwners.get(i) != FREE_PAGE && isPageFree(i)) {
-                setPageOwner(i, FREE_PAGE);
+            int owner = pageOwners.get(i);
+            if (owner != FREE_PAGE && isPageFree(i)) {
+                owners.add(ObjectUtils.getInteger(owner));
+                freePages.add(i);
             }
         }
+        // now, for each current owner, remove those
+        // this is much faster than removing them individually 
+        // as this would cause O(n^2) behavior
+        for (Iterator it = owners.iterator(); it.hasNext();) {
+            int owner = ((Integer) it.next()).intValue();
+            database.getStorage(owner, this).removePages(freePages);
+        }
+        // now free up the pages
+        for (int i = 0; i < freePages.size(); i++) {
+            int idx = freePages.get(i);
+            setPageOwner(idx, FREE_PAGE);
+        }
     }
 
     /**

h2/src/main/org/h2/store/Storage.java

@@ -131,7 +131,7 @@ public class Storage {
                 int page = file.getPage(next);
                 if (lastCheckedPage != page) {
                     if (pageIndex < 0) {
-                        pageIndex = pages.findNextValueIndex(page);
+                        pageIndex = pages.findNextIndexSorted(page);
                     } else {
                         pageIndex++;
                     }
@@ -352,13 +352,25 @@ public class Storage {
         pages.addValueSorted(i);
     }
 
+    /**
+     * Remove a list of page from this storage.
+     * 
+     * @param removeSorted the pages to remove
+     */
+    void removePages(IntArray removeSorted) {
+        pages.removeAllSorted(removeSorted);
+    }
+
     /**
      * Remove a page from this storage.
      * 
      * @param i the page to remove
      */
     void removePage(int i) {
-        pages.removeValue(i);
+        int idx = pages.findIndexSorted(i);
+        if (idx != -1) {
+            pages.remove(idx);
+        }
     }
 
     private void checkOnePage() throws SQLException {

h2/src/main/org/h2/util/IntArray.java

@@ -201,6 +201,44 @@ public class IntArray {
         throw Message.getInternalError();
     }
     
+    /**
+     * Remove the last element of this list that matches this value.
+     * 
+     * @param value the value to be remove
+     */
+    public void removeLastValue(int value) {
+        for (int i = size - 1; i >= 0; i--) {
+            if (data[i] == value) {
+                remove(i);
+                return;
+            }
+        }
+        throw Message.getInternalError();
+    }    
+    
+    /**
+     * Return the index with a this value.
+     * If the list is not sorted, the result of this operation is undefined.
+     * 
+     * @param value the value to find
+     * @return the index or -1 if not found
+     */
+    public int findIndexSorted(int value) {
+        int l = 0, r = size;
+        while (l < r) {
+            int i = (l + r) >>> 1;
+            int d = data[i];
+            if (d == value) {
+                return i;
+            } else if (d > value) {
+                r = i;
+            } else {
+                l = i + 1;
+            }
+        }
+        return -1;
+    }
+    
     /**
      * Return the next index with a value larger than this one.
      * If the list is not sorted, the result of this operation is undefined.
@@ -208,7 +246,7 @@ public class IntArray {
      * @param value the value to find
      * @return the index
      */
-    public int findNextValueIndex(int value) {
+    public int findNextIndexSorted(int value) {
         int l = 0, r = size;
         while (l < r) {
             int i = (l + r) >>> 1;
@@ -220,13 +258,6 @@ public class IntArray {
             }
         }
         return l;
-
-//        for(int i=0; i<size; i++) {
-//            if(data[i] >= value) {
-//                return i;
-//            }
-//        }
-//        return size;
     }
 
     /**
@@ -251,6 +282,24 @@ public class IntArray {
     public void toArray(int[] array) {
         System.arraycopy(data, 0, array, 0, size);
     }
+    
+    /**
+     * Remove all values from the given sorted list from this sorted list.
+     * 
+     * @param removeSorted the value to remove
+     */    
+    public void removeAllSorted(IntArray removeSorted) {
+        int[] d = new int[data.length];
+        int newSize = 0;
+        for (int i = 0; i < size; i++) {
+            int old = data[i];
+            if (removeSorted.findIndexSorted(old) == -1) {
+                d[newSize++] = old;
+            }
+        }
+        data = d;
+        size = newSize;
+    }
 
 //    ArrayList data = new ArrayList();
 //

h2/src/test/org/h2/test/TestAll.java

@@ -265,9 +265,14 @@ java org.h2.test.TestAll timer
         System.setProperty("h2.maxMemoryRowsDistinct", "128");
 
 /*
-        
+
+move DataPage.createByteArray to ByteUtils, improve message
+
 run the performance tests as part of the unit test
 
+switch to JDK 1.6 by default
+integrate java queries
+
 jazoon
 
 H2 Console should support Java Queries