MVStore: encrypted stores are now supported - table engine

h2/src/main/org/h2/engine/Database.java

@@ -2402,4 +2402,8 @@ public class Database implements DataHandler {
         throw DbException.throwInternalError();
     }
 
+    public byte[] getFilePasswordHash() {
+        return filePasswordHash;
+    }
+
 }

h2/src/main/org/h2/mvstore/DataUtils.java

@@ -512,7 +512,7 @@ public class DataUtils {
         for (int i = 0, size = s.length(); i < size;) {
             int startKey = i;
             i = s.indexOf(':', i);
-            checkArgument(i > 0, "Not a map");
+            checkArgument(i >= 0, "Not a map");
             String key = s.substring(startKey, i++);
             StringBuilder buff = new StringBuilder();
             while (i < size) {

h2/src/main/org/h2/mvstore/MVStore.java

@@ -24,7 +24,7 @@ import org.h2.mvstore.cache.CacheLongKeyLIRS;
 import org.h2.mvstore.cache.FilePathCache;
 import org.h2.mvstore.type.StringDataType;
 import org.h2.store.fs.FilePath;
-import org.h2.store.fs.FilePathCrypt2;
+import org.h2.store.fs.FilePathCrypt;
 import org.h2.store.fs.FileUtils;
 import org.h2.util.MathUtils;
 import org.h2.util.New;
@@ -43,27 +43,21 @@ H:3,...
 
 TODO:
 
-- file system encryption (
-  test and document speed,
-  support un-aligned operations,
-  test other algorithms)
+- automated 'kill process' and 'power failure' test
+- test stream store if data doesn't fit in memory
 - mvcc with multiple transactions
 - update checkstyle
-- automated 'kill process' and 'power failure' test
 - maybe split database into multiple files, to speed up compact
 - auto-compact from time to time and on close
 - test and possibly improve compact operation (for large dbs)
-- performance test with encrypting file system
 - possibly split chunk data into immutable and mutable
 - compact: avoid processing pages using a counting bloom filter
 - defragment (re-creating maps, specially those with small pages)
 - remove DataType.getMaxLength (use ByteArrayOutputStream or getMemory)
 - chunk header: store changed chunk data as row; maybe after the root
 - chunk checksum (header, last page, 2 bytes per page?)
-- file locking: solve problem that locks are shared for a VM
-- store file "header" at the end of each chunk; at the end of the file
 - is there a better name for the file header,
--- if it's no longer always at the beginning of a file?
+-- if it's no longer always at the beginning of a file? store header?
 - on insert, if the child page is already full, don't load and modify it
 -- split directly (for leaves with 1 entry)
 - maybe let a chunk point to possible next chunks
@@ -80,7 +74,7 @@ TODO:
 - chunk metadata: do not store default values
 - support maps without values (just existence of the key)
 - support maps without keys (counted b-tree features)
-- use a small object cache (StringCache)
+- use a small object cache (StringCache), test on Android
 - dump values
 - tool to import / manipulate CSV files (maybe concurrently)
 - map split / merge (fast if no overlap)
@@ -96,7 +90,10 @@ TODO:
 -- to support concurrent updates and writes, and very large maps
 - implement an off-heap file system
 - remove change cursor, or add support for writing to branches
-- file encryption / decryption using multiple threads
+- file encryption: try using multiple threads
+- file encryption: support un-aligned operations
+- file encryption: separate algorithm/key for tweak
+- file encryption: add a fast (insecure) algorithm
 
 */
 
@@ -408,7 +405,7 @@ public class MVStore {
             file = f.open(readOnly ? "r" : "rw");
             if (filePassword != null) {
                 byte[] password = DataUtils.getPasswordBytes(filePassword);
-                file = new FilePathCrypt2.FileCrypt2(password, file);
+                file = new FilePathCrypt.FileCrypt2(password, file);
             }
             file = FilePathCache.wrap(file);
             if (readOnly) {

h2/src/main/org/h2/mvstore/Page.java

@@ -501,10 +501,11 @@ public class Page {
             sharedFlags &= ~SHARED_KEYS;
         }
         Object old = keys[index];
+        DataType keyType = map.getKeyType();
         if (old != null) {
-            memory -= map.getKeyType().getMemory(old);
+            memory -= keyType.getMemory(old);
         }
-        memory += map.getKeyType().getMemory(key);
+        memory += keyType.getMemory(key);
         keys[index] = key;
     }
 
@@ -521,8 +522,9 @@ public class Page {
             values = Arrays.copyOf(values, values.length);
             sharedFlags &= ~SHARED_VALUES;
         }
-        memory -= map.getValueType().getMemory(old);
-        memory += map.getValueType().getMemory(value);
+        DataType valueType = map.getValueType();
+        memory -= valueType.getMemory(old);
+        memory += valueType.getMemory(value);
         values[index] = value;
         return old;
     }
@@ -772,8 +774,9 @@ public class Page {
                 : DataUtils.PAGE_TYPE_LEAF;
         buff.put((byte) type);
         int compressStart = buff.position();
+        DataType keyType = map.getKeyType();
         for (int i = 0; i < len; i++) {
-            map.getKeyType().write(buff, keys[i]);
+            keyType.write(buff, keys[i]);
         }
         if (type == DataUtils.PAGE_TYPE_NODE) {
             for (int i = 0; i <= len; i++) {
@@ -783,8 +786,9 @@ public class Page {
                 DataUtils.writeVarLong(buff, counts[i]);
             }
         } else {
+            DataType valueType = map.getValueType();
             for (int i = 0; i < len; i++) {
-                map.getValueType().write(buff, values[i]);
+                valueType.write(buff, values[i]);
             }
         }
         if (map.getStore().getCompress()) {
@@ -839,12 +843,14 @@ public class Page {
         int maxLength = 4 + 2 + DataUtils.MAX_VAR_INT_LEN
                 + DataUtils.MAX_VAR_INT_LEN + 1;
         int len = keyCount;
+        DataType keyType = map.getKeyType();
         for (int i = 0; i < len; i++) {
-            maxLength += map.getKeyType().getMaxLength(keys[i]);
+            maxLength += keyType.getMaxLength(keys[i]);
         }
         if (isLeaf()) {
+            DataType valueType = map.getValueType();
             for (int i = 0; i < len; i++) {
-                maxLength += map.getValueType().getMaxLength(values[i]);
+                maxLength += valueType.getMaxLength(values[i]);
             }
         } else {
             maxLength += 8 * len;
@@ -912,12 +918,14 @@ public class Page {
 
     private int calculateMemory() {
         int mem = DataUtils.PAGE_MEMORY;
+        DataType keyType = map.getKeyType();
         for (int i = 0; i < keyCount; i++) {
-            mem += map.getKeyType().getMemory(keys[i]);
+            mem += keyType.getMemory(keys[i]);
         }
         if (this.isLeaf()) {
+            DataType valueType = map.getValueType();
             for (int i = 0; i < keyCount; i++) {
-                mem += map.getValueType().getMemory(values[i]);
+                mem += valueType.getMemory(values[i]);
             }
         } else {
             mem += this.getChildPageCount() * DataUtils.PAGE_MEMORY_CHILD;

h2/src/main/org/h2/mvstore/db/MVTableEngine.java

@@ -49,6 +49,7 @@ public class MVTableEngine implements TableEngine {
     @Override
     public TableBase createTable(CreateTableData data) {
         Database db = data.session.getDatabase();
+        byte[] key = db.getFilePasswordHash();
         String storeName = db.getDatabasePath();
         MVStore.Builder builder = new MVStore.Builder();
         Store store;
@@ -62,6 +63,13 @@ public class MVTableEngine implements TableEngine {
                     if (db.isReadOnly()) {
                         builder.readOnly();
                     }
+                    if (key != null) {
+                        char[] password = new char[key.length];
+                        for (int i = 0; i < key.length; i++) {
+                            password[i] = (char) key[i];
+                        }
+                        builder.encryptionKey(password);
+                    }
                     store = new Store(db, builder.open());
                     STORES.put(storeName, store);
                 } else if (store.db != db) {

h2/src/main/org/h2/store/fs/FilePathCrypt2.java → h2/src/main/org/h2/store/fs/FilePathCrypt.java

@@ -24,7 +24,7 @@ import org.h2.util.StringUtils;
 /**
  * An encrypted file.
  */
-public class FilePathCrypt2 extends FilePathWrapper {
+public class FilePathCrypt extends FilePathWrapper {
 
     private static final String SCHEME = "crypt2";
 
@@ -32,7 +32,7 @@ public class FilePathCrypt2 extends FilePathWrapper {
      * Register this file system.
      */
     public static void register() {
-        FilePath.register(new FilePathCrypt2());
+        FilePath.register(new FilePathCrypt());
     }
 
     public FileChannel open(String mode) throws IOException {
@@ -121,12 +121,13 @@ public class FilePathCrypt2 extends FilePathWrapper {
         /**
          * The length of the salt, in bytes.
          */
-        private static final int SALT_LENGTH = 32;
+        private static final int SALT_LENGTH = 8;
 
         /**
-         * The number of iterations.
+         * The number of iterations. It is relatively low; a higher value would
+         * slow down opening files on Android too much.
          */
-        private static final int HASH_ITERATIONS = 10000;
+        private static final int HASH_ITERATIONS = 10;
 
         private final FileChannel base;
 

h2/src/test/org/h2/test/store/TestMVTableEngine.java

@@ -7,6 +7,7 @@ package org.h2.test.store;
 
 import java.math.BigDecimal;
 import java.sql.Connection;
+import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -33,6 +34,7 @@ public class TestMVTableEngine extends TestBase {
     }
 
     public void test() throws Exception {
+        testEncryption();
         testReadOnly();
         testReuseDiskSpace();
         testDataTypes();
@@ -40,6 +42,26 @@ public class TestMVTableEngine extends TestBase {
         testSimple();
     }
 
+    private void testEncryption() throws Exception {
+        FileUtils.deleteRecursive(getBaseDir(), true);
+        String dbName = "mvstore" +
+                ";DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine";
+        Connection conn;
+        Statement stat;
+        String url = getURL(dbName + ";CIPHER=AES", true);
+        String user = "sa";
+        String password = "123 123";
+        conn = DriverManager.getConnection(url, user, password);
+        stat = conn.createStatement();
+        stat.execute("create table test(id int)");
+        conn.close();
+        conn = DriverManager.getConnection(url, user, password);
+        stat = conn.createStatement();
+        stat.execute("select * from test");
+        conn.close();
+        FileUtils.deleteRecursive(getBaseDir(), true);
+    }
+
     private void testReadOnly() throws Exception {
         FileUtils.deleteRecursive(getBaseDir(), true);
         String dbName = "mvstore" +

h2/src/test/org/h2/test/unit/TestFileSystem.java

@@ -14,8 +14,8 @@ import java.io.OutputStream;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
 import java.nio.channels.FileChannel.MapMode;
+import java.nio.channels.FileLock;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
@@ -23,7 +23,6 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.List;
 import java.util.Random;
-import org.h2.dev.fs.FilePathCrypt;
 import org.h2.message.DbException;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FileUtils;
@@ -58,9 +57,8 @@ public class TestFileSystem extends TestBase {
         testUnsupportedFeatures(getBaseDir());
         testMemFsDir();
         testClasspath();
-        FilePathCrypt.register();
         FilePathDebug.register().setTrace(true);
-        testFileSystem("crypt:aes:x:" + getBaseDir() + "/fs");
+        // testFileSystem("crypt:007:" + getBaseDir() + "/fs");
 
         testSimpleExpandTruncateSize();
         testSplitDatabaseInZip();
@@ -74,8 +72,7 @@ public class TestFileSystem extends TestBase {
         testFileSystem("memLZF:");
         testUserHome();
         try {
-            FilePathCrypt.register();
-            testFileSystem("crypt:aes:x:" + getBaseDir() + "/fs");
+            // testFileSystem("crypt:007:" + getBaseDir() + "/fs");
             testFileSystem("nio:" + getBaseDir() + "/fs");
             testFileSystem("nioMapped:" + getBaseDir() + "/fs");
             if (!config.splitFileSystem) {