Merge branch 'master' of https://github.com/h2database/h2database

h2/src/docsrc/html/changelog.html

@@ -33,6 +33,14 @@ Change Log
 </li>
 <li>Issue #295: JdbcResultSet.getObject(int, Class) returns null instead of throwing.
 </li>
+<li>Mac OS X: Console tool process did not stop on exit.
+</li>
+<li>MVStoreTool: add "repair" feature.
+</li>
+<li>Garbage collection of unused chunks should be faster still.
+</li>
+<li>MVStore / transaction store: opening a store in read-only mode does no longer loop.
+</li>
 </ul>
 
 <h2>Version 1.4.192 Beta (2016-05-26)</h2>

h2/src/main/org/h2/mvstore/DataUtils.java

@@ -171,11 +171,6 @@ public class DataUtils {
      */
     private static final byte[] EMPTY_BYTES = {};
 
-    /**
-     * The maximum byte to grow a buffer at a time.
-     */
-    private static final int MAX_GROW = 16 * 1024 * 1024;
-
     /**
      * Get the length of the variable size int.
      *
@@ -300,14 +295,12 @@ public class DataUtils {
     /**
      * Write characters from a string (without the length).
      *
-     * @param buff the target buffer
+     * @param buff the target buffer (must be large enough)
      * @param s the string
      * @param len the number of characters
-     * @return the byte buffer
      */
-    public static ByteBuffer writeStringData(ByteBuffer buff,
+    public static void writeStringData(ByteBuffer buff,
             String s, int len) {
-        buff = DataUtils.ensureCapacity(buff, 3 * len);
         for (int i = 0; i < len; i++) {
             int c = s.charAt(i);
             if (c < 0x80) {
@@ -321,7 +314,6 @@ public class DataUtils {
                 buff.put((byte) (c & 0x3f));
             }
         }
-        return buff;
     }
 
     /**
@@ -860,32 +852,6 @@ public class DataUtils {
         }
     }
 
-    /**
-     * Ensure the byte buffer has the given capacity, plus 1 KB. If not, a new,
-     * larger byte buffer is created and the data is copied.
-     *
-     * @param buff the byte buffer
-     * @param len the minimum remaining capacity
-     * @return the byte buffer (possibly a new one)
-     */
-    public static ByteBuffer ensureCapacity(ByteBuffer buff, int len) {
-        len += 1024;
-        if (buff.remaining() > len) {
-            return buff;
-        }
-        return grow(buff, len);
-    }
-
-    private static ByteBuffer grow(ByteBuffer buff, int len) {
-        len = buff.remaining() + len;
-        int capacity = buff.capacity();
-        len = Math.max(len, Math.min(capacity + MAX_GROW, capacity * 2));
-        ByteBuffer temp = ByteBuffer.allocate(len);
-        buff.flip();
-        temp.put(buff);
-        return temp;
-    }
-
     /**
      * Read a hex long value from a map.
      *

h2/src/main/org/h2/mvstore/MVStore.java

@@ -1056,8 +1056,11 @@ public class MVStore {
         long time = getTimeSinceCreation();
         int freeDelay = retentionTime / 10;
         if (time >= lastFreeUnusedChunks + freeDelay) {
+            // set early in case it fails (out of memory or so)
             lastFreeUnusedChunks = time;
             freeUnusedChunks();
+            // set it here as well, to avoid calling it often if it was slow
+            lastFreeUnusedChunks = getTimeSinceCreation();
         }
         int currentUnsavedPageCount = unsavedMemory;
         long storeVersion = currentStoreVersion;
@@ -1514,6 +1517,9 @@ public class MVStore {
      * @param minPercent the minimum percentage to save
      */
     private void shrinkFileIfPossible(int minPercent) {
+        if (fileStore.isReadOnly()) {
+            return;
+        }
         long end = getFileLengthInUse();
         long fileSize = fileStore.size();
         if (end >= fileSize) {
@@ -2660,6 +2666,15 @@ public class MVStore {
         return cache;
     }
 
+    /**
+     * Whether the store is read-only.
+     *
+     * @return true if it is
+     */
+    public boolean isReadOnly() {
+        return fileStore == null ? false : fileStore.isReadOnly();
+    }
+
     /**
      * A background writer thread to automatically store changes from time to
      * time.

h2/src/main/org/h2/mvstore/MVStoreTool.java

@@ -6,6 +6,7 @@
 package org.h2.mvstore;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.io.Writer;
 import java.nio.ByteBuffer;
@@ -60,6 +61,9 @@ public class MVStoreTool {
             } else if ("-compress".equals(args[i])) {
                 String fileName = args[++i];
                 compact(fileName, true);
+            } else if ("-repair".equals(args[i])) {
+                String fileName = args[++i];
+                repair(fileName);
             }
         }
     }
@@ -325,12 +329,13 @@ public class MVStoreTool {
      *
      * @param fileName the name of the file
      * @param writer the print writer
+     * @return null if successful (if there was no error), otherwise the error message
      */
-    public static void info(String fileName, Writer writer) {
+    public static String info(String fileName, Writer writer) {
         PrintWriter pw = new PrintWriter(writer, true);
         if (!FilePath.get(fileName).exists()) {
             pw.println("File not found: " + fileName);
-            return;
+            return "File not found: " + fileName;
         }
         long fileLength = FileUtils.size(fileName);
         MVStore store = new MVStore.Builder().
@@ -389,10 +394,12 @@ public class MVStoreTool {
         } catch (Exception e) {
             pw.println("ERROR: " + e);
             e.printStackTrace(pw);
+            return e.getMessage();
         } finally {
             store.close();
         }
         pw.flush();
+        return null;
     }
 
     private static String formatTimestamp(long t, long start) {
@@ -518,6 +525,137 @@ public class MVStoreTool {
         }
     }
 
+    /**
+     * Repair a store by rolling back to the newest good version.
+     *
+     * @param fileName the file name
+     */
+    public static void repair(String fileName) {
+        PrintWriter pw = new PrintWriter(System.out);
+        long version = Long.MAX_VALUE;
+        OutputStream ignore = new OutputStream() {
+            @Override
+            public void write(int b) throws IOException {
+                // ignore
+            }
+        };
+        while (version >= 0) {
+            pw.println(version == Long.MAX_VALUE ? "Trying latest version" : ("Trying version " + version));
+            pw.flush();
+            version = rollback(fileName, version, new PrintWriter(ignore));
+            try {
+                String error = info(fileName + ".temp", new PrintWriter(ignore));
+                if (error == null) {
+                    FilePath.get(fileName).moveTo(FilePath.get(fileName + ".back"), true);
+                    FilePath.get(fileName + ".temp").moveTo(FilePath.get(fileName), true);
+                    pw.println("Success");
+                    break;
+                }
+                pw.println("    ... failed: " + error);
+            } catch (Exception e) {
+                pw.println("Fail: " + e.getMessage());
+                pw.flush();
+            }
+            version--;
+        }
+        pw.flush();
+    }
+
+    /**
+     * Roll back to a given revision into a a file called *.temp.
+     *
+     * @param fileName the file name
+     * @param targetVersion the version to roll back to (Long.MAX_VALUE for the latest version)
+     * @return the version rolled back to (-1 if no version)
+     */
+    public static long rollback(String fileName, long targetVersion, Writer writer) {
+        long newestVersion = -1;
+        PrintWriter pw = new PrintWriter(writer, true);
+        if (!FilePath.get(fileName).exists()) {
+            pw.println("File not found: " + fileName);
+            return newestVersion;
+        }
+        FileChannel file = null;
+        FileChannel target = null;
+        int blockSize = MVStore.BLOCK_SIZE;
+        try {
+            file = FilePath.get(fileName).open("r");
+            FilePath.get(fileName + ".temp").delete();
+            target = FilePath.get(fileName + ".temp").open("rw");
+            long fileSize = file.size();
+            ByteBuffer block = ByteBuffer.allocate(4096);
+            Chunk newestChunk = null;
+            for (long pos = 0; pos < fileSize;) {
+                block.rewind();
+                DataUtils.readFully(file, pos, block);
+                block.rewind();
+                int headerType = block.get();
+                if (headerType == 'H') {
+                    block.rewind();
+                    target.write(block, pos);
+                    pos += blockSize;
+                    continue;
+                }
+                if (headerType != 'c') {
+                    pos += blockSize;
+                    continue;
+                }
+                Chunk c = null;
+                try {
+                    c = Chunk.readChunkHeader(block, pos);
+                } catch (IllegalStateException e) {
+                    pos += blockSize;
+                    continue;
+                }
+                if (c.len <= 0) {
+                    // not a chunk
+                    pos += blockSize;
+                    continue;
+                }
+                int length = c.len * MVStore.BLOCK_SIZE;
+                ByteBuffer chunk = ByteBuffer.allocate(length);
+                DataUtils.readFully(file, pos, chunk);
+                if (c.version > targetVersion) {
+                    // newer than the requested version
+                    pos += length;
+                    continue;
+                }
+                chunk.rewind();
+                target.write(chunk, pos);
+                if (newestChunk == null || c.version > newestChunk.version) {
+                    newestChunk = c;
+                    newestVersion = c.version;
+                }
+                pos += length;
+            }
+            int length = newestChunk.len * MVStore.BLOCK_SIZE;
+            ByteBuffer chunk = ByteBuffer.allocate(length);
+            DataUtils.readFully(file, newestChunk.block * MVStore.BLOCK_SIZE, chunk);
+            chunk.rewind();
+            target.write(chunk, fileSize);
+        } catch (IOException e) {
+            pw.println("ERROR: " + e);
+            e.printStackTrace(pw);
+        } finally {
+            if (file != null) {
+                try {
+                    file.close();
+                } catch (IOException e) {
+                    // ignore
+                }
+            }
+            if (target != null) {
+                try {
+                    target.close();
+                } catch (IOException e) {
+                    // ignore
+                }
+            }
+        }
+        pw.flush();
+        return newestVersion;
+    }
+
     /**
      * A data type that can read any data that is persisted, and converts it to
      * a byte array.

h2/src/main/org/h2/mvstore/WriteBuffer.java

@@ -73,19 +73,7 @@ public class WriteBuffer {
      */
     public WriteBuffer putStringData(String s, int len) {
         ByteBuffer b = ensureCapacity(3 * len);
-        for (int i = 0; i < len; i++) {
-            int c = s.charAt(i);
-            if (c < 0x80) {
-                b.put((byte) c);
-            } else if (c >= 0x800) {
-                b.put((byte) (0xe0 | (c >> 12)));
-                b.put((byte) (((c >> 6) & 0x3f)));
-                b.put((byte) (c & 0x3f));
-            } else {
-                b.put((byte) (0xc0 | (c >> 6)));
-                b.put((byte) (c & 0x3f));
-            }
-        }
+        DataUtils.writeStringData(b, s, len);
         return this;
     }
 

h2/src/main/org/h2/mvstore/db/TransactionStore.java

@@ -1246,6 +1246,10 @@ public class TransactionStore {
                 Object[] d;
                 d = transaction.store.undoLog.get(id);
                 if (d == null) {
+                    if (transaction.store.store.isReadOnly()) {
+                        // uncommitted transaction for a read-only store
+                        return null;
+                    }
                     // this entry should be committed or rolled back
                     // in the meantime (the transaction might still be open)
                     // or it might be changed again in a different

h2/src/main/org/h2/mvstore/rtree/MVRTreeMap.java

@@ -17,7 +17,8 @@ import org.h2.mvstore.type.ObjectDataType;
 import org.h2.util.New;
 
 /**
- * An r-tree implementation. It uses the quadratic split algorithm.
+ * An r-tree implementation. It supports both the linear and the quadratic split
+ * algorithm.
  *
  * @param <V> the value class
  */

h2/src/main/org/h2/tools/Console.java

@@ -351,9 +351,10 @@ ShutdownHandler {
                 trayIconUsed = false;
             }
             System.gc();
+            // Mac OS X: Console tool process did not stop on exit
+            System.exit(0);
         }
 //*/
-        // System.exit(0);
     }
 
 //## AWT ##

h2/src/main/org/h2/util/Profiler.java

@@ -156,7 +156,7 @@ public class Profiler implements Runnable {
                         sumClasses = true;
                     } else if ("-methods".equals(arg)) {
                         sumMethods = true;
-                    } else if ("-packages".equals(args)) {
+                    } else if ("-packages".equals(arg)) {
                         sumClasses = false;
                         sumMethods = false;
                     } else {

h2/src/test/org/h2/test/store/CalculateHashConstant.java

@@ -18,8 +18,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.h2.security.AES;
 
 /**
- * Calculate the constant for the secondary hash function, so that the hash
- * function mixes the input bits as much as possible.
+ * Calculate the constant for the secondary / supplemental hash function, so
+ * that the hash function mixes the input bits as much as possible.
  */
 public class CalculateHashConstant implements Runnable {
 
@@ -227,6 +227,30 @@ public class CalculateHashConstant implements Runnable {
         System.out.println("Collisions: " + collisions);
     }
 
+    /**
+     * Calculate the multiplicative inverse of a value (int).
+     *
+     * @param a the value
+     * @return the multiplicative inverse
+     */
+    static long calcMultiplicativeInverse(long a) {
+        return BigInteger.valueOf(a).modPow(
+                BigInteger.valueOf((1 << 31) - 1), BigInteger.valueOf(1L << 32)).longValue();
+    }
+
+    /**
+     * Calculate the multiplicative inverse of a value (long).
+     *
+     * @param a the value
+     * @return the multiplicative inverse
+     */
+    static long calcMultiplicativeInverseLong(long a) {
+        BigInteger oneShift64 = BigInteger.valueOf(1).shiftLeft(64);
+        BigInteger oneShift63 = BigInteger.valueOf(1).shiftLeft(63);
+        return BigInteger.valueOf(a).modPow(
+                oneShift63.subtract(BigInteger.ONE),
+                oneShift64).longValue();
+    }
     /**
      * Store a random file to be analyzed by the Diehard test.
      */