A persistent multi-version map: open the file in exclusive or shared mode; allow…

A persistent multi-version map: open the file in exclusive or shared mode; allow to read and update the file header.

h2/src/main/org/h2/store/fs/FilePathDisk.java

@@ -405,7 +405,7 @@ class FileDisk extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
-        return file.getChannel().tryLock();
+        return file.getChannel().tryLock(position, size, shared);
     }
 
     public void implCloseChannel() throws IOException {

h2/src/main/org/h2/store/fs/FilePathMem.java

@@ -216,7 +216,7 @@ class FilePathMemLZF extends FilePathMem {
  */
 class FileMem extends FileBase {
 
-    private final FileMemData data;
+    final FileMemData data;
     private final boolean readOnly;
     private long pos;
 
@@ -282,8 +282,29 @@ class FileMem extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
+        if (shared) {
+            if (!data.lockShared()) {
                 return null;
             }
+        } else {
+            if (!data.lockExclusive()) {
+                return null;
+            }
+        }
+        FileLock lock = new FileLock(null, position, size, shared) {
+
+            @Override
+            public boolean isValid() {
+                return true;
+            }
+
+            @Override
+            public void release() throws IOException {
+                data.unlock();
+            }
+        };
+        return lock;
+    }
 
     public String toString() {
         return data.getName();
@@ -314,6 +335,8 @@ class FileMemData {
     private byte[][] data;
     private long lastModified;
     private boolean isReadOnly;
+    private boolean isLockedExclusive;
+    private int sharedLockCount;
 
     static {
         byte[] n = new byte[BLOCK_SIZE];
@@ -322,6 +345,37 @@ class FileMemData {
         System.arraycopy(BUFFER, 0, COMPRESSED_EMPTY_BLOCK, 0, len);
     }
 
+    FileMemData(String name, boolean compress) {
+        this.name = name;
+        this.compress = compress;
+        data = new byte[0][];
+        lastModified = System.currentTimeMillis();
+    }
+
+    synchronized boolean lockExclusive() {
+        if (sharedLockCount > 0 || isLockedExclusive) {
+            return false;
+        }
+        isLockedExclusive = true;
+        return true;
+    }
+
+    synchronized boolean lockShared() {
+        if (isLockedExclusive) {
+            return false;
+        }
+        sharedLockCount++;
+        return true;
+    }
+
+    synchronized void unlock() {
+        if (isLockedExclusive) {
+            isLockedExclusive = false;
+        } else {
+            sharedLockCount = Math.max(0, sharedLockCount - 1);
+        }
+    }
+
     /**
      * This small cache compresses the data if an element leaves the cache.
      */
@@ -371,13 +425,7 @@ class FileMemData {
             }
             return false;
         }
-    }
 
-    FileMemData(String name, boolean compress) {
-        this.name = name;
-        this.compress = compress;
-        data = new byte[0][];
-        lastModified = System.currentTimeMillis();
     }
 
     private static void compressLater(byte[][] data, int page) {

h2/src/main/org/h2/store/fs/FilePathNio.java

@@ -98,7 +98,7 @@ class FileNio extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
-        return channel.tryLock();
+        return channel.tryLock(position, size, shared);
     }
 
     public String toString() {

h2/src/main/org/h2/store/fs/FilePathNioMapped.java

@@ -233,7 +233,7 @@ class FileNioMapped extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
-        return file.getChannel().tryLock();
+        return file.getChannel().tryLock(position, size, shared);
     }
 
 }

h2/src/main/org/h2/store/fs/FilePathRec.java

@@ -172,7 +172,7 @@ class FileRec extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
-        return channel.tryLock();
+        return channel.tryLock(position, size, shared);
     }
 
 }

h2/src/main/org/h2/store/fs/FilePathSplit.java

@@ -370,7 +370,7 @@ class FileSplit extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
-        return list[0].tryLock();
+        return list[0].tryLock(position, size, shared);
     }
 
     public String toString() {

h2/src/main/org/h2/store/fs/FilePathZip.java

@@ -316,6 +316,19 @@ class FileZip extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
+        if (shared) {
+            return new FileLock(null, position, size, shared) {
+
+                @Override
+                public boolean isValid() {
+                    return true;
+                }
+
+                @Override
+                public void release() throws IOException {
+                    // ignore
+                }};
+        }
         return null;
     }
 

h2/src/test/org/h2/test/store/TestConcurrent.java

@@ -10,6 +10,7 @@ import java.util.Map;
 import java.util.Random;
 import org.h2.dev.store.btree.MVMap;
 import org.h2.dev.store.btree.MVStore;
+import org.h2.dev.store.btree.MVStoreBuilder;
 import org.h2.test.TestBase;
 import org.h2.util.Task;
 
@@ -34,7 +35,7 @@ public class TestConcurrent extends TestMVStore {
     }
 
     private void testConcurrentIterate() {
-        MVStore s = MVStore.open(null, new TestMapFactory());
+        MVStore s = MVStoreBuilder.inMemory().with(new TestMapFactory()).open();
         s.setMaxPageSize(3);
         final MVMap<Integer, Integer> map = s.openMap("test");
         final int len = 10;

h2/src/test/org/h2/test/store/TestDataUtils.java

@@ -8,6 +8,7 @@ package org.h2.test.store;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 import java.util.HashMap;
 import org.h2.dev.store.btree.DataUtils;
 import org.h2.test.TestBase;
@@ -27,6 +28,7 @@ public class TestDataUtils extends TestBase {
     }
 
     public void test() throws Exception {
+        testFletcher();
         testMap();
         testMaxShortVarIntVarLong();
         testVarIntVarLong();
@@ -35,6 +37,29 @@ public class TestDataUtils extends TestBase {
         testEncodeLength();
     }
 
+    private void testFletcher() throws Exception {
+        byte[] data = new byte[10000];
+        for (int i = 0; i < 10000; i += 1000) {
+            assertEquals(-1, DataUtils.getFletcher32(data, i));
+        }
+        Arrays.fill(data, (byte) 255);
+        for (int i = 0; i < 10000; i += 1000) {
+            assertEquals(-1, DataUtils.getFletcher32(data, i));
+        }
+        long last = 0;
+        for (int i = 1; i < 255; i++) {
+            Arrays.fill(data, (byte) i);
+            for (int j = 0; j < 10; j += 2) {
+                int x = DataUtils.getFletcher32(data, j);
+                assertTrue(x != last);
+                last = x;
+            }
+        }
+        Arrays.fill(data, (byte) 10);
+        assertEquals(0x1e1e1414, DataUtils.getFletcher32(data, 10000));
+        assertEquals(0x1e3fa7ed, DataUtils.getFletcher32("Fletcher32".getBytes(), 10));
+    }
+
     private void testMap() {
         StringBuilder buff = new StringBuilder();
         DataUtils.appendMap(buff,  "", "");

h2/src/test/org/h2/test/store/TestMVRTree.java

@@ -64,7 +64,7 @@ public class TestMVRTree extends TestMVStore {
             SpatialKey k = new SpatialKey(i, x - p, x + p, y - p, y + p);
             r.add(k, "" + i);
             if (i > 0 && (i % len / 10) == 0) {
-                s.save();
+                s.store();
             }
             if (i > 0 && (i % 10000) == 0) {
                 render(r, getBaseDir() + "/test.png");
@@ -72,7 +72,7 @@ public class TestMVRTree extends TestMVStore {
         }
         // System.out.println(prof.getTop(5));
         // System.out.println("add: " + (System.currentTimeMillis() - t));
-        s.save();
+        s.store();
         s.close();
         s = openStore(fileName);
         r = s.openMap("data", "r", "s2", "");

h2/src/test/org/h2/test/store/TestMapFactory.java

@@ -5,8 +5,6 @@
  */
 package org.h2.test.store;
 
-import org.h2.compress.CompressLZF;
-import org.h2.compress.Compressor;
 import org.h2.dev.store.btree.MVMap;
 import org.h2.dev.store.btree.MVStore;
 import org.h2.dev.store.btree.DataType;
@@ -59,9 +57,4 @@ public class TestMapFactory implements MapFactory {
         throw new RuntimeException("Unsupported object class " + objectClass.toString());
     }
 
-    @Override
-    public Compressor buildCompressor() {
-        return new CompressLZF();
-    }
-
 }

h2/src/test/org/h2/test/utils/FilePathDebug.java

@@ -293,7 +293,7 @@ class FileDebug extends FileBase {
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
         debug("tryLock");
-        return channel.tryLock();
+        return channel.tryLock(position, size, shared);
     }
 
 }

h2/src/test/org/h2/test/utils/FilePathUnstable.java

@@ -209,7 +209,7 @@ class FileUnstable extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
-        return channel.tryLock();
+        return channel.tryLock(position, size, shared);
     }
 
 }
\ No newline at end of file

h2/src/tools/org/h2/dev/fs/FilePathCrypt.java

@@ -188,7 +188,7 @@ class FileCrypt extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
-        return file.tryLock();
+        return file.tryLock(position, size, shared);
     }
 
     public void implCloseChannel() throws IOException {

h2/src/tools/org/h2/dev/fs/FilePathZip2.java

@@ -397,6 +397,19 @@ class FileZip2 extends FileBase {
     }
 
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
+        if (shared) {
+            return new FileLock(null, position, size, shared) {
+
+                @Override
+                public boolean isValid() {
+                    return true;
+                }
+
+                @Override
+                public void release() throws IOException {
+                    // ignore
+                }};
+        }
         return null;
     }
 

h2/src/tools/org/h2/dev/store/btree/DataUtils.java

@@ -11,6 +11,8 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import org.h2.util.New;
 
@@ -425,7 +427,23 @@ public class DataUtils {
     }
 
     /**
-     * Append a key-value pair to the string buffer. Keys may not contain a
+     * Append a map to the string builder, sorted by key.
+     *
+     * @param buff the target buffer
+     * @param map the map
+     * @return the string builder
+     */
+    public static StringBuilder appendMap(StringBuilder buff, HashMap<String, ?> map) {
+        ArrayList<String> list = New.arrayList(map.keySet());
+        Collections.sort(list);
+        for (String k : list) {
+            appendMap(buff, k, map.get(k));
+        }
+        return buff;
+    }
+
+    /**
+     * Append a key-value pair to the string builder. Keys may not contain a
      * colon. Values that contain a comma or a double quote are enclosed in
      * double quotes, with special characters escaped using a backslash.
      *
@@ -490,4 +508,26 @@ public class DataUtils {
         return map;
     }
 
+    /**
+     * Calculate the Fletcher32 checksum.
+     *
+     * @param bytes the bytes
+     * @param length the message length (must be a multiple of 2)
+     * @return the checksum
+     */
+    public static int getFletcher32(byte[] bytes, int length) {
+        int s1 = 0xffff, s2 = 0xffff;
+        for (int i = 0; i < length;) {
+            for (int end = Math.min(i + 718, length); i < end;) {
+                int x = ((bytes[i++] & 0xff) << 8) | (bytes[i++] & 0xff);
+                s2 += s1 += x;
+            }
+            s1 = (s1 & 0xffff) + (s1 >>> 16);
+            s2 = (s2 & 0xffff) + (s2 >>> 16);
+        }
+        s1 = (s1 & 0xffff) + (s1 >>> 16);
+        s2 = (s2 & 0xffff) + (s2 >>> 16);
+        return (s2 << 16) | s1;
+    }
+
 }

h2/src/tools/org/h2/dev/store/btree/MVStoreBuilder.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.dev.store.btree;
+
+import java.util.HashMap;
+import org.h2.util.New;
+
+/**
+ * A builder for an MVStore.
+ */
+public class MVStoreBuilder {
+
+    private HashMap<String, Object> config = New.hashMap();
+
+    /**
+     * Use the following file name. If the file does not exist, it is
+     * automatically created.
+     *
+     * @param fileName the file name
+     * @return this
+     */
+    public static MVStoreBuilder fileBased(String fileName) {
+        return new MVStoreBuilder().fileName(fileName);
+    }
+
+    /**
+     * Open the store in-memory. In this case, no data may be saved.
+     *
+     * @return the store
+     */
+    public static MVStoreBuilder inMemory() {
+        return new MVStoreBuilder();
+    }
+
+    private MVStoreBuilder set(String key, Object value) {
+        if (config.containsKey(key)) {
+            throw new IllegalArgumentException("Parameter " + config.get(key) + " is already set");
+        }
+        config.put(key, value);
+        return this;
+    }
+
+    private MVStoreBuilder fileName(String fileName) {
+        return set("fileName", fileName);
+    }
+
+    /**
+     * Open the file in read-only mode. In this case, a shared lock will be
+     * acquired to ensure the file is not concurrently opened in write mode.
+     * <p>
+     * If this option is not used, the file is locked exclusively.
+     * <p>
+     * Please note a store may only be opened once in every JVM (no matter
+     * whether it is opened in read-only or read-write mode), because each file
+     * may be locked only once in a process.
+     *
+     * @return this
+     */
+    public MVStoreBuilder readOnly() {
+        return set("openMode", "r");
+    }
+
+    public MVStoreBuilder with(MapFactory mapFactory) {
+        int todoRemove;
+        return set("mapFactory", mapFactory);
+    }
+
+    /**
+     * Open the store.
+     *
+     * @return the opened store
+     */
+    public MVStore open() {
+        MVStore s = new MVStore(config);
+        s.open();
+        return s;
+    }
+
+    public String toString() {
+        return DataUtils.appendMap(new StringBuilder(), config).toString();
+    }
+
+    public static MVStoreBuilder fromString(String s) {
+        HashMap<String, String> config = DataUtils.parseMap(s);
+        MVStoreBuilder builder = new MVStoreBuilder();
+        builder.config.putAll(config);
+        return builder;
+    }
+
+}

h2/src/tools/org/h2/dev/store/btree/Dump.java → h2/src/tools/org/h2/dev/store/btree/MVStoreUtils.java

@@ -17,61 +17,58 @@ import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FileUtils;
 
 /**
- * Convert a database file to a human-readable text dump.
+ * Utility methods used in combination with the MVStore.
  */
-public class Dump {
-
-    private static int blockSize = 4 * 1024;
+public class MVStoreUtils {
 
     /**
      * Runs this tool.
      * Options are case sensitive. Supported options are:
      * <table>
-     * <tr><td>[-file]</td>
-     * <td>The database file name</td></tr>
+     * <tr><td>[-dump &lt;dir&gt;]</td>
+     * <td>Dump the contends of the file</td></tr>
      * </table>
      *
      * @param args the command line arguments
      */
-    public static void main(String... args) {
-        String fileName = "test.h3";
+    public static void main(String... args) throws IOException {
         for (int i = 0; i < args.length; i++) {
-            if ("-file".equals(args[i])) {
-                fileName = args[++i];
+            if ("-dump".equals(args[i])) {
+                String fileName = args[++i];
+                dump(fileName, new PrintWriter(System.out));
             }
         }
-        dump(fileName, new PrintWriter(System.out));
     }
 
     /**
-     * Dump the contents of the file.
+     * Read the contents of the file and display them in a human-readable
+     * format.
      *
      * @param fileName the name of the file
      * @param writer the print writer
      */
-    public static void dump(String fileName, PrintWriter writer) {
+    public static void dump(String fileName, PrintWriter writer) throws IOException {
         if (!FileUtils.exists(fileName)) {
             writer.println("File not found: " + fileName);
             return;
         }
         FileChannel file = null;
+        int blockSize = MVStore.BLOCK_SIZE;
         try {
             file = FilePath.get(fileName).open("r");
             long fileLength = file.size();
-            file.position(0);
             byte[] header = new byte[blockSize];
-            file.read(ByteBuffer.wrap(header));
+            file.read(ByteBuffer.wrap(header), 0);
             Properties prop = new Properties();
             prop.load(new ByteArrayInputStream(header));
             prop.load(new StringReader(new String(header, "UTF-8")));
             writer.println("file " + fileName);
             writer.println("    length " + fileLength);
             writer.println("    " + prop);
-            ByteBuffer block = ByteBuffer.wrap(new byte[32]);
+            ByteBuffer block = ByteBuffer.allocate(32);
             for (long pos = 0; pos < fileLength;) {
-                file.position(pos);
                 block.rewind();
-                FileUtils.readFully(file,  block);
+                DataUtils.readFully(file, pos, block);
                 block.rewind();
                 if (block.get() != 'c') {
                     pos += blockSize;
@@ -89,8 +86,7 @@ public class Dump {
                         " root " + metaRootPos +
                         " maxLengthLive " + maxLengthLive);
                 ByteBuffer chunk = ByteBuffer.allocate(chunkLength);
-                file.position(pos);
-                FileUtils.readFully(file, chunk);
+                DataUtils.readFully(file, pos, chunk);
                 int p = block.position();
                 pos = (pos + chunkLength + blockSize) / blockSize * blockSize;
                 chunkLength -= p;
@@ -114,6 +110,7 @@ public class Dump {
             }
         } catch (IOException e) {
             writer.println("ERROR: " + e);
+            throw e;
         } finally {
             if (file != null) {
                 try {

h2/src/tools/org/h2/dev/store/btree/MapFactory.java

@@ -6,7 +6,6 @@
  */
 package org.h2.dev.store.btree;
 
-import org.h2.compress.Compressor;
 
 /**
  * A factory for maps and data types.
@@ -37,13 +36,6 @@ public interface MapFactory {
      */
     DataType buildDataType(String dataType);
 
-    /**
-     * Create a new compressor.
-     *
-     * @return the compressor
-     */
-    Compressor buildCompressor();
-
     /**
      * Get the data type object for the given class.
      *

h2/src/tools/org/h2/dev/store/btree/Page.java

@@ -130,12 +130,12 @@ public class Page {
             maxLength = (int) Math.min(fileSize - filePos, maxLength);
             int length = maxLength;
             if (maxLength == Integer.MAX_VALUE) {
-                buff = ByteBuffer.wrap(new byte[128]);
+                buff = ByteBuffer.allocate(128);
                 DataUtils.readFully(file, filePos, buff);
                 maxLength = buff.getInt();
                 //read the first bytes again
             }
-            buff = ByteBuffer.wrap(new byte[length]);
+            buff = ByteBuffer.allocate(length);
             DataUtils.readFully(file, filePos, buff);
         } catch (IOException e) {
             throw new RuntimeException(e);
@@ -664,9 +664,9 @@ public class Page {
             int compLen = pageLength + start - buff.position();
             byte[] comp = new byte[compLen];
             buff.get(comp);
-            byte[] exp = new byte[compLen + lenAdd];
-            compressor.expand(comp, 0, compLen, exp, 0, exp.length);
-            buff = ByteBuffer.wrap(exp);
+            int l = compLen + lenAdd;
+            buff = ByteBuffer.allocate(l);
+            compressor.expand(comp, 0, compLen, buff.array(), 0, l);
         }
         DataType keyType = map.getKeyType();
         for (int i = 0; i < len; i++) {