MVStore: encrypted stores are now supported.

h2/src/docsrc/html/changelog.html

@@ -33,6 +33,8 @@ Change Log
 </li><li>New connection setting "DEFAULT_TABLE_ENGINE" to use a specific
     table engine if none is set explicitly. This is to simplify testing 
     the MVStore table engine.
+</li><li>MVStore: encrypted stores are now supported.
+    Only standardized algorithms are used: PBKDF2, SHA-256, XTS-AES, AES-128. 
 </li><li>MVStore: improved API thanks to Simo Tripodi.
 </li><li>MVStore: maps can now be renamed.
 </li><li>MVStore: store the file header also at the end of each chunk,

h2/src/docsrc/html/mvstore.html

@@ -43,7 +43,7 @@ But it can be also directly within an application, without using JDBC or SQL.
 </li><li>Transactions (even if they are persisted) can be rolled back.
 </li><li>The tool is very modular. It supports pluggable data types / serialization,
 pluggable map implementations (B-tree, R-tree, concurrent B-tree currently), BLOB storage,
-and a file system abstraction to support encryption and compressed read-only files.
+and a file system abstraction to support encryption and zip files.
 </li></ul>
 
 <h2 id="example_code">Example Code</h2>
@@ -325,6 +325,29 @@ new data is always appended at the end of the file.
 Then, the file can be copied (the file handle is available to the application).
 </p>
 
+<h3>Encrypted Files</h3>
+<p>
+Data can be encrypted as follows:
+</p>
+<pre>
+MVStore s = new MVStore.Builder().
+    fileName(fileName).
+    encryptionKey("007".toCharArray()).
+    open();
+</pre>
+<p>
+The same security algorithms are used as modern disk encryption software use.
+The password char array is cleared after use, 
+to reduce the risk that the password is stolen
+even if the attacker has access to the main memory.
+The password is hashed using the PBKDF2 standard, using the SHA-256 hash algorithm.
+The length of the salt is 256 bits, so that an attacker can not use a rainbow table.
+The salt is generated using a cryptographically secure random number generator.
+The number of PBKDF2 iterations is 10000, so that an attacker can not "brute force" the password.
+The file itself is encrypted using the standardized disk encryption mode XTS-AES,
+so that only little more than one AES-128 round per block is needed.
+</p>
+
 <h3>Tools</h3>
 <p>
 There is a tool (<code>MVStoreTool</code>) to dump the contents of a file.

h2/src/main/org/h2/mvstore/DataUtils.java

@@ -512,6 +512,7 @@ public class DataUtils {
         for (int i = 0, size = s.length(); i < size;) {
             int startKey = i;
             i = s.indexOf(':', i);
+            checkArgument(i > 0, "Not a map");
             String key = s.substring(startKey, i++);
             StringBuilder buff = new StringBuilder();
             while (i < size) {
@@ -608,4 +609,16 @@ public class DataUtils {
                 Constants.VERSION_MINOR + "." + Constants.BUILD_ID + "]";
     }
 
+    static byte[] getPasswordBytes(char[] passwordChars) {
+        // using UTF-16
+        int len = passwordChars.length;
+        byte[] password = new byte[len * 2];
+        for (int i = 0; i < len; i++) {
+            char c = passwordChars[i];
+            password[i + i] = (byte) (c >>> 8);
+            password[i + i + 1] = (byte) c;
+        }
+        return password;
+    }
+
 }

h2/src/main/org/h2/mvstore/MVStore.java

@@ -11,6 +11,7 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Collections;
 import java.util.Comparator;
@@ -23,6 +24,7 @@ import org.h2.mvstore.cache.CacheLongKeyLIRS;
 import org.h2.mvstore.cache.FilePathCache;
 import org.h2.mvstore.type.StringDataType;
 import org.h2.store.fs.FilePath;
+import org.h2.store.fs.FilePathCrypt2;
 import org.h2.store.fs.FileUtils;
 import org.h2.util.MathUtils;
 import org.h2.util.New;
@@ -41,8 +43,7 @@ H:3,...
 
 TODO:
 
-- file system encryption: check standard
-
+- file system encryption
 - mvcc with multiple transactions
 - update checkstyle
 - automated 'kill process' and 'power failure' test
@@ -115,6 +116,7 @@ public class MVStore {
     private static final int FORMAT_READ = 1;
 
     private final String fileName;
+    private final char[] filePassword;
 
     private int pageSize = 6 * 1024;
 
@@ -185,8 +187,10 @@ public class MVStore {
             int mb = s == null ? 16 : Integer.parseInt(s.toString());
             cache = new CacheLongKeyLIRS<Page>(
                     mb * 1024 * 1024, 2048, 16, mb * 1024 * 1024 / 2048 * 2 / 100);
+            filePassword = (char[]) config.get("encrypt");
         } else {
             cache = null;
+            filePassword = null;
         }
     }
 
@@ -368,12 +372,18 @@ public class MVStore {
             return;
         }
         FileUtils.createDirectories(FileUtils.getParent(fileName));
+        try {
             if (readOnly) {
                 openFile();
             } else if (!openFile()) {
                 readOnly = true;
                 openFile();
             }
+        } finally {
+            if (filePassword != null) {
+                Arrays.fill(filePassword, (char) 0);
+            }
+        }
     }
 
     /**
@@ -391,7 +401,12 @@ public class MVStore {
             if (f.exists() && !f.canWrite()) {
                 readOnly = true;
             }
-            file = FilePathCache.wrap(f.open(readOnly ? "r" : "rw"));
+            file = f.open(readOnly ? "r" : "rw");
+            if (filePassword != null) {
+                byte[] password = DataUtils.getPasswordBytes(filePassword);
+                file = new FilePathCrypt2.FileCrypt2(password, file);
+            }
+            file = FilePathCache.wrap(file);
             if (readOnly) {
                 fileLock = file.tryLock(0, Long.MAX_VALUE, true);
                 if (fileLock == null) {
@@ -433,7 +448,7 @@ public class MVStore {
             }
         } catch (Exception e) {
             try {
-                close();
+                close(false);
             } catch (Exception e2) {
                 // ignore
             }
@@ -443,6 +458,7 @@ public class MVStore {
         return true;
     }
 
+
     private void readMeta() {
         Chunk header = readChunkHeader(rootChunkStart);
         lastChunkId = header.id;
@@ -545,10 +561,16 @@ public class MVStore {
      * Close the file. Uncommitted changes are ignored, and all open maps are closed.
      */
     public void close() {
+        close(true);
+    }
+
+    private void close(boolean shrinkIfPossible) {
         closed = true;
         if (file != null) {
             try {
+                if (shrinkIfPossible) {
                     shrinkFileIfPossible(0);
+                }
                 log("file close");
                 if (fileLock != null) {
                     fileLock.release();
@@ -1416,6 +1438,21 @@ public class MVStore {
             return set("fileName", fileName);
         }
 
+        /**
+         * Encrypt / decrypt the file using the given password. This method has
+         * no effect for in-memory stores. The password is passed as a char
+         * array so that it can be cleared as soon as possible. Please note
+         * there is still a small risk that password stays in memory (due to
+         * Java garbage collection). Also, the hashed encryption key is kept in
+         * memory as long as the file is open.
+         *
+         * @param password the password
+         * @return this
+         */
+        public Builder encryptionKey(char[] password) {
+            return set("encrypt", password);
+        }
+
         /**
          * Open the file in read-only mode. In this case, a shared lock will be
          * acquired to ensure the file is not concurrently opened in write mode.

h2/src/main/org/h2/security/SHA256.java

@@ -77,6 +77,54 @@ public class SHA256 {
         return getHash(buff, true);
     }
 
+    public static byte[] getHMAC(byte[] key, byte[] message) {
+        int blockSize = 64;
+        if (key.length > blockSize) {
+            key = getHash(key, false);
+        }
+        if (key.length < blockSize) {
+            key = Arrays.copyOf(key, blockSize);
+        }
+        byte[] iKeyPadMessage = new byte[blockSize + message.length];
+        Arrays.fill(iKeyPadMessage, 0, blockSize, (byte) 0x36);
+        xor(iKeyPadMessage, key, blockSize);
+        System.arraycopy(message, 0, iKeyPadMessage, blockSize, message.length);
+        byte[] k = getHash(iKeyPadMessage, false);
+        byte[] oKeyPad = new byte[blockSize + k.length];
+        Arrays.fill(oKeyPad, 0, blockSize, (byte) 0x5c);
+        xor(oKeyPad, key, blockSize);
+        System.arraycopy(k, 0, oKeyPad, blockSize, k.length);
+        return getHash(oKeyPad, false);
+    }
+
+    private static void xor(byte[] target, byte[] data, int len) {
+        for (int i = 0; i < len; i++) {
+            target[i] ^= data[i];
+        }
+    }
+
+    public static byte[] getPBKDF2(byte[] password, byte[] salt, int iterations, int len) {
+        byte[] result = new byte[len];
+        byte[] last = null;
+        for (int k = 1, offset = 0; offset < len; k++, offset += 32) {
+            for (int i = 0; i < iterations; i++) {
+                byte[] x;
+                if (i == 0) {
+                    x = Arrays.copyOf(salt, salt.length + 4);
+                    writeInt(x, salt.length, k);
+                } else {
+                    x = last;
+                }
+                last = getHMAC(password, x);
+                for (int j = 0; j < 32 && j + offset < len; j++) {
+                    result[j + offset] ^= last[j];
+                }
+            }
+        }
+        Arrays.fill(password, (byte) 0);
+        return result;
+    }
+
     /**
      * Calculate the hash code for the given data.
      *

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -40,6 +40,7 @@ public class TestMVStore extends TestBase {
 
     public void test() throws Exception {
         FileUtils.deleteRecursive(getBaseDir(), true);
+        testEncryptedFile();
         testFileFormatChange();
         testRecreateMap();
         testRenameMapRollback();
@@ -74,6 +75,54 @@ public class TestMVStore extends TestBase {
         testSimple();
     }
 
+    private void testEncryptedFile() {
+        String fileName = getBaseDir() + "/testEncryptedFile.h3";
+        FileUtils.delete(fileName);
+        MVStore s;
+        MVMap<Integer, String> m;
+
+        char[] passwordChars = "007".toCharArray();
+        s = new MVStore.Builder().
+                fileName(fileName).
+                encryptionKey(passwordChars).
+                open();
+        assertEquals(0, passwordChars[0]);
+        assertEquals(0, passwordChars[1]);
+        assertEquals(0, passwordChars[2]);
+        assertTrue(FileUtils.exists(fileName));
+        m = s.openMap("test");
+        m.put(1, "Hello");
+        assertEquals("Hello", m.get(1));
+        s.store();
+        s.close();
+
+        passwordChars = "008".toCharArray();
+        try {
+            s = new MVStore.Builder().
+                    fileName(fileName).
+                    encryptionKey(passwordChars).open();
+            fail();
+        } catch (IllegalStateException e) {
+            assertTrue(e.getCause() != null);
+        }
+        assertEquals(0, passwordChars[0]);
+        assertEquals(0, passwordChars[1]);
+        assertEquals(0, passwordChars[2]);
+
+        passwordChars = "007".toCharArray();
+        s = new MVStore.Builder().
+                fileName(fileName).
+                encryptionKey(passwordChars).open();
+        assertEquals(0, passwordChars[0]);
+        assertEquals(0, passwordChars[1]);
+        assertEquals(0, passwordChars[2]);
+        m = s.openMap("test");
+        assertEquals("Hello", m.get(1));
+        s.close();
+        FileUtils.delete(fileName);
+        assertFalse(FileUtils.exists(fileName));
+    }
+
     private void testFileFormatChange() {
         String fileName = getBaseDir() + "/testFileFormatChange.h3";
         FileUtils.delete(fileName);

h2/src/test/org/h2/test/unit/TestSecurity.java

@@ -46,9 +46,71 @@ public class TestSecurity extends TestBase {
     }
 
     private void testSHA() {
+        testPBKDF2();
+        testHMAC();
         testOneSHA();
     }
 
+    private void testPBKDF2() {
+        // test vectors from StackOverflow (PBKDF2-HMAC-SHA2)
+        assertEquals(
+                "120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b",
+                StringUtils.convertBytesToHex(
+                SHA256.getPBKDF2(
+                "password".getBytes(),
+                "salt".getBytes(), 1, 32)));
+        assertEquals(
+                "ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43",
+                StringUtils.convertBytesToHex(
+                SHA256.getPBKDF2(
+                "password".getBytes(),
+                "salt".getBytes(), 2, 32)));
+        assertEquals(
+                "c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a",
+                StringUtils.convertBytesToHex(
+                SHA256.getPBKDF2(
+                "password".getBytes(),
+                "salt".getBytes(), 4096, 32)));
+        // take a very long time to calculate
+        // assertEquals(
+        //         "cf81c66fe8cfc04d1f31ecb65dab4089f7f179e89b3b0bcb17ad10e3ac6eba46",
+        //         StringUtils.convertBytesToHex(
+        //         SHA256.getPBKDF2(
+        //         "password".getBytes(),
+        //         "salt".getBytes(), 16777216, 32)));
+        assertEquals(
+                "348c89dbcbd32b2f32d814b8116e84cf2b17347ebc1800181c4e2a1fb8dd53e1c635518c7dac47e9",
+                StringUtils.convertBytesToHex(
+                SHA256.getPBKDF2(
+                "passwordPASSWORDpassword".getBytes(),
+                "saltSALTsaltSALTsaltSALTsaltSALTsalt".getBytes(), 4096, 40)));
+        assertEquals(
+                "89b69d0516f829893c696226650a8687",
+                StringUtils.convertBytesToHex(
+                SHA256.getPBKDF2(
+                "pass\0word".getBytes(),
+                "sa\0lt".getBytes(), 4096, 16)));
+
+        // the password is filled with zeroes
+        byte[] password = "Test".getBytes();
+        SHA256.getPBKDF2(password, "".getBytes(), 1, 16);
+        assertEquals(new byte[4], password);
+    }
+
+    private void testHMAC() {
+        // from Wikipedia
+        assertEquals(
+                "b613679a0814d9ec772f95d778c35fc5ff1697c493715653c6c712144292c5ad",
+                StringUtils.convertBytesToHex(
+                        SHA256.getHMAC(new byte[0], new byte[0])));
+        assertEquals(
+                "f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8",
+                StringUtils.convertBytesToHex(
+                SHA256.getHMAC(
+                "key".getBytes(),
+                "The quick brown fox jumps over the lazy dog".getBytes())));
+    }
+
     private String getHashString(byte[] data) {
         byte[] result = SHA256.getHash(data, true);
         if (data.length > 0) {