MVStore: encrypted stores are now supported.

h2/src/docsrc/html/mvstore.html

@@ -34,23 +34,23 @@ MVStore
 The MVStore is work in progress, and is planned to be the next storage subsystem of H2.
 But it can be also directly within an application, without using JDBC or SQL.
 </p>
-<ul><li>MVStore stands for multi-version store.
+<ul><li>MVStore stands for "multi-version store".
 </li><li>Each store contains a number of maps (using the <code>java.util.Map</code> interface).
-</li><li>Both file based persistence and in-memory operation are supported.
+</li><li>Both file-based persistence and in-memory operation are supported.
 </li><li>It is intended to be fast, simple to use, and small.
 </li><li>Old versions of the data can be read concurrently with all other operations.
 </li><li>Transaction are supported (currently only one transaction at a time).
 </li><li>Transactions (even if they are persisted) can be rolled back.
 </li><li>The tool is very modular. It supports pluggable data types / serialization,
 pluggable map implementations (B-tree, R-tree, concurrent B-tree currently), BLOB storage,
-and a file system abstraction to support encryption and zip files.
+and a file system abstraction to support encrypted files and zip files.
 </li></ul>
 
 <h2 id="example_code">Example Code</h2>
 <h3>Map Operations and Versioning</h3>
 <p>
 The following sample code show how to create a store,
-open a map, add some data, and access the current and an old version.
+open a map, add some data, and access the current and an old version:
 </p>
 <pre>
 import org.h2.mvstore.*;
@@ -73,7 +73,7 @@ s.incrementVersion();
 
 // more changes, in the new version
 // changes can be rolled back if required
-// changes always go into 'head' (the newest version)
+// changes always go into "head" (the newest version)
 map.put(1, "Hi");
 map.remove(2);
 
@@ -86,7 +86,7 @@ s.store();
 
 // print the old version (can be done
 // concurrently with further modifications)
-// this will print Hello World
+// this will print "Hello" and "World":
 System.out.println(oldMap.get(1));
 System.out.println(oldMap.get(2));
 oldMap.close();
@@ -101,11 +101,12 @@ s.close();
 <h3>Store Builder</h3>
 <p>
 The <code>MVStore.Builder</code> provides a fluid interface
-to build a store if more complex configuration options are used.
+to build a store if more complex configuration options are used:
 </p>
 <pre>
 MVStore s = new MVStore.Builder().
     fileName(fileName).
+    encryptionKey("007".toCharArray()).
     cacheSizeMB(10).
     readOnly().
     open();
@@ -114,7 +115,7 @@ MVStore s = new MVStore.Builder().
 <h3>R-Tree</h3>
 <p>
 The <code>MVRTreeMap</code> is an R-tree implementation
-that supports fast spatial queries.
+that supports fast spatial queries. It can be used as follows:
 </p>
 <pre>
 // create an in-memory store
@@ -327,6 +328,7 @@ Then, the file can be copied (the file handle is available to the application).
 
 <h3>Encrypted Files</h3>
 <p>
+File encryption ensures the data can only be read with the correct password.
 Data can be encrypted as follows:
 </p>
 <pre>
@@ -336,17 +338,23 @@ MVStore s = new MVStore.Builder().
     open();
 </pre>
 <p>
-The same security algorithms are used as modern disk encryption software use.
-The password char array is cleared after use, 
-to reduce the risk that the password is stolen
-even if the attacker has access to the main memory.
-The password is hashed using the PBKDF2 standard, using the SHA-256 hash algorithm.
-The length of the salt is 256 bits, so that an attacker can not use a rainbow table.
-The salt is generated using a cryptographically secure random number generator.
-The number of PBKDF2 iterations is 10000, so that an attacker can not "brute force" the password.
-The file itself is encrypted using the standardized disk encryption mode XTS-AES,
-so that only little more than one AES-128 round per block is needed.
+</p><p>
+Only state of the art disk encryption algorithms are used:
 </p>
+<ul><li>The password char array is cleared after use, 
+    to reduce the risk that the password is stolen
+    even if the attacker has access to the main memory.
+</li><li>The password is hashed according to the PBKDF2 standard, 
+    using the SHA-256 hash algorithm.
+</li><li>The length of the salt is 256 bits, 
+    so that an attacker can not use a pre-calculated password hash table (rainbow table).
+</li><li>The salt is generated using a cryptographically secure random number generator.
+</li><li>To protect against brute-force password cracking attacks, 
+    the number of PBKDF2 iterations is 10000.
+</li><li>For fastest possible read and write operations, 
+    the file itself is encrypted using the standardized disk encryption mode XTS-AES.
+    This means only little more than one AES-128 round per block is needed.
+</li></ul>
 
 <h3>Tools</h3>
 <p>

h2/src/main/org/h2/mvstore/MVStore.java

@@ -43,7 +43,10 @@ H:3,...
 
 TODO:
 
-- file system encryption
+- file system encryption (
+  test and document speed,
+  support un-aligned operations,
+  test other algorithms)
 - mvcc with multiple transactions
 - update checkstyle
 - automated 'kill process' and 'power failure' test
@@ -93,6 +96,7 @@ TODO:
 -- to support concurrent updates and writes, and very large maps
 - implement an off-heap file system
 - remove change cursor, or add support for writing to branches
+- file encryption / decryption using multiple threads
 
 */
 

h2/src/main/org/h2/store/fs/FilePathCrypt2.java

@@ -16,6 +16,7 @@ import java.util.Arrays;
 import org.h2.message.DbException;
 import org.h2.mvstore.DataUtils;
 import org.h2.security.AES;
+import org.h2.security.BlockCipher;
 import org.h2.security.SHA256;
 import org.h2.util.MathUtils;
 import org.h2.util.StringUtils;
@@ -129,7 +130,7 @@ public class FilePathCrypt2 extends FilePathWrapper {
 
         private final FileChannel base;
 
-        private final XTSAES xts;
+        private final XTS xts;
 
         private long pos;
 
@@ -147,8 +148,11 @@ public class FilePathCrypt2 extends FilePathWrapper {
                 salt = new byte[SALT_LENGTH];
                 DataUtils.readFully(base, SALT_POS, ByteBuffer.wrap(salt));
             }
-            byte[] key = SHA256.getPBKDF2(passwordBytes, salt, HASH_ITERATIONS, 16);
-            xts = new XTSAES(key);
+            // TODO support Fog (and maybe Fog2)
+            // Fog cipher = new Fog();
+            AES cipher = new AES();
+            cipher.setKey(SHA256.getPBKDF2(passwordBytes, salt, HASH_ITERATIONS, 16));
+            xts = new XTS(cipher);
         }
 
         protected void implCloseChannel() throws IOException {
@@ -249,12 +253,12 @@ public class FilePathCrypt2 extends FilePathWrapper {
     }
 
     /**
-     * An XTS-AES implementation as described in
+     * An XTS implementation as described in
      * IEEE P1619 (Standard Architecture for Encrypted Shared Storage Media).
      * See also
      * http://axelkenzo.ru/downloads/1619-2007-NIST-Submission.pdf
      */
-    static class XTSAES {
+    static class XTS {
 
         /**
          * Galois Field feedback.
@@ -264,75 +268,75 @@ public class FilePathCrypt2 extends FilePathWrapper {
         /**
          * The AES encryption block size.
          */
-        private static final int AES_BLOCK_SIZE = 16;
+        private static final int CIPHER_BLOCK_SIZE = 16;
 
-        private final AES aes = new AES();
+        private final BlockCipher cipher;
 
-        XTSAES(byte[] key) {
-            aes.setKey(key);
+        XTS(BlockCipher cipher) {
+            this.cipher = cipher;
         }
 
         void encrypt(long id, int len, byte[] data, int offset) {
             byte[] tweak = initTweak(id);
             int i = 0;
-            for (; i + AES_BLOCK_SIZE <= len; i += AES_BLOCK_SIZE) {
+            for (; i + CIPHER_BLOCK_SIZE <= len; i += CIPHER_BLOCK_SIZE) {
                 if (i > 0) {
                     updateTweak(tweak);
                 }
                 xorTweak(data, i + offset, tweak);
-                aes.encrypt(data, i + offset, AES_BLOCK_SIZE);
+                cipher.encrypt(data, i + offset, CIPHER_BLOCK_SIZE);
                 xorTweak(data, i + offset, tweak);
             }
             if (i < len) {
                 updateTweak(tweak);
-                swap(data, i + offset, i - AES_BLOCK_SIZE + offset, len - i);
-                xorTweak(data, i - AES_BLOCK_SIZE + offset, tweak);
-                aes.encrypt(data, i - AES_BLOCK_SIZE + offset, AES_BLOCK_SIZE);
-                xorTweak(data, i - AES_BLOCK_SIZE + offset, tweak);
+                swap(data, i + offset, i - CIPHER_BLOCK_SIZE + offset, len - i);
+                xorTweak(data, i - CIPHER_BLOCK_SIZE + offset, tweak);
+                cipher.encrypt(data, i - CIPHER_BLOCK_SIZE + offset, CIPHER_BLOCK_SIZE);
+                xorTweak(data, i - CIPHER_BLOCK_SIZE + offset, tweak);
             }
         }
 
         void decrypt(long id, int len, byte[] data, int offset) {
             byte[] tweak = initTweak(id), tweakEnd = tweak;
             int i = 0;
-            for (; i + AES_BLOCK_SIZE <= len; i += AES_BLOCK_SIZE) {
+            for (; i + CIPHER_BLOCK_SIZE <= len; i += CIPHER_BLOCK_SIZE) {
                 if (i > 0) {
                     updateTweak(tweak);
-                    if (i + AES_BLOCK_SIZE + AES_BLOCK_SIZE > len && i + AES_BLOCK_SIZE < len) {
-                        tweakEnd = Arrays.copyOf(tweak, AES_BLOCK_SIZE);
+                    if (i + CIPHER_BLOCK_SIZE + CIPHER_BLOCK_SIZE > len && i + CIPHER_BLOCK_SIZE < len) {
+                        tweakEnd = Arrays.copyOf(tweak, CIPHER_BLOCK_SIZE);
                         updateTweak(tweak);
                     }
                 }
                 xorTweak(data, i + offset, tweak);
-                aes.decrypt(data, i + offset, AES_BLOCK_SIZE);
+                cipher.decrypt(data, i + offset, CIPHER_BLOCK_SIZE);
                 xorTweak(data, i + offset, tweak);
             }
             if (i < len) {
-                swap(data, i, i - AES_BLOCK_SIZE + offset, len - i + offset);
-                xorTweak(data, i - AES_BLOCK_SIZE  + offset, tweakEnd);
-                aes.decrypt(data, i - AES_BLOCK_SIZE + offset, AES_BLOCK_SIZE);
-                xorTweak(data, i - AES_BLOCK_SIZE + offset, tweakEnd);
+                swap(data, i, i - CIPHER_BLOCK_SIZE + offset, len - i + offset);
+                xorTweak(data, i - CIPHER_BLOCK_SIZE  + offset, tweakEnd);
+                cipher.decrypt(data, i - CIPHER_BLOCK_SIZE + offset, CIPHER_BLOCK_SIZE);
+                xorTweak(data, i - CIPHER_BLOCK_SIZE + offset, tweakEnd);
             }
         }
 
         private byte[] initTweak(long id) {
-            byte[] tweak = new byte[AES_BLOCK_SIZE];
-            for (int j = 0; j < AES_BLOCK_SIZE; j++, id >>>= 8) {
+            byte[] tweak = new byte[CIPHER_BLOCK_SIZE];
+            for (int j = 0; j < CIPHER_BLOCK_SIZE; j++, id >>>= 8) {
                 tweak[j] = (byte) (id & 0xff);
             }
-            aes.encrypt(tweak, 0, AES_BLOCK_SIZE);
+            cipher.encrypt(tweak, 0, CIPHER_BLOCK_SIZE);
             return tweak;
         }
 
         private static void xorTweak(byte[] data, int pos, byte[] tweak) {
-            for (int i = 0; i < AES_BLOCK_SIZE; i++) {
+            for (int i = 0; i < CIPHER_BLOCK_SIZE; i++) {
                 data[pos + i] ^= tweak[i];
             }
         }
 
         static void updateTweak(byte[] tweak) {
             byte ci = 0, co = 0;
-            for (int i = 0; i < AES_BLOCK_SIZE; i++) {
+            for (int i = 0; i < CIPHER_BLOCK_SIZE; i++) {
                 co = (byte) ((tweak[i] >> 7) & 1);
                 tweak[i] = (byte) (((tweak[i] << 1) + ci) & 255);
                 ci = co;

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -507,7 +507,7 @@ public class TestMVStore extends TestBase {
 
         // more changes, in the new version
         // changes can be rolled back if required
-        // changes always go into 'head' (the newest version)
+        // changes always go into "head" (the newest version)
         map.put(1, "Hi");
         map.remove(2);
 
@@ -520,7 +520,7 @@ public class TestMVStore extends TestBase {
 
         // print the old version (can be done
         // concurrently with further modifications)
-        // this will print Hello World
+        // this will print "Hello" and "World":
         // System.out.println(oldMap.get(1));
         assertEquals("Hello", oldMap.get(1));
         // System.out.println(oldMap.get(2));