Skip to content

H
h2database
提交 42af75c5 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

MVStore: encrypted stores are now supported.

上级 4ee69cc5
显示空白字符变更
内嵌 并排
正在显示 包含 600 行增加 和 11 行删除
h2/src/docsrc/html/changelog.html
浏览文件 @ 42af75c5
......@@ -33,6 +33,8 @@ Change Log
</li><li>New connection setting "DEFAULT_TABLE_ENGINE" to use a specific
table engine if none is set explicitly. This is to simplify testing
the MVStore table engine.
</li><li>MVStore: encrypted stores are now supported.
Only standardized algorithms are used: PBKDF2, SHA-256, XTS-AES, AES-128.
</li><li>MVStore: improved API thanks to Simo Tripodi.
</li><li>MVStore: maps can now be renamed.
</li><li>MVStore: store the file header also at the end of each chunk,
......
h2/src/docsrc/html/mvstore.html
浏览文件 @ 42af75c5
......@@ -43,7 +43,7 @@ But it can be also directly within an application, without using JDBC or SQL.
</li><li>Transactions (even if they are persisted) can be rolled back.
</li><li>The tool is very modular. It supports pluggable data types / serialization,
pluggable map implementations (B-tree, R-tree, concurrent B-tree currently), BLOB storage,
and a file system abstraction to support encryption and compressed read-only files.
and a file system abstraction to support encryption and zip files.
</li></ul>
<h2 id="example_code">Example Code</h2>
......@@ -325,6 +325,29 @@ new data is always appended at the end of the file.
Then, the file can be copied (the file handle is available to the application).
</p>
<h3>Encrypted Files</h3>
<p>
Data can be encrypted as follows:
</p>
<pre>
MVStore s = new MVStore.Builder().
fileName(fileName).
encryptionKey("007".toCharArray()).
open();
</pre>
<p>
The same security algorithms are used as modern disk encryption software use.
The password char array is cleared after use,
to reduce the risk that the password is stolen
even if the attacker has access to the main memory.
The password is hashed using the PBKDF2 standard, using the SHA-256 hash algorithm.
The length of the salt is 256 bits, so that an attacker can not use a rainbow table.
The salt is generated using a cryptographically secure random number generator.
The number of PBKDF2 iterations is 10000, so that an attacker can not "brute force" the password.
The file itself is encrypted using the standardized disk encryption mode XTS-AES,
so that only little more than one AES-128 round per block is needed.
</p>
<h3>Tools</h3>
<p>
There is a tool (<code>MVStoreTool</code>) to dump the contents of a file.
......
h2/src/main/org/h2/mvstore/DataUtils.java
浏览文件 @ 42af75c5
......@@ -512,6 +512,7 @@ public class DataUtils {
for (int i = 0, size = s.length(); i < size;) {
int startKey = i;
i = s.indexOf(':', i);
checkArgument(i > 0, "Not a map");
String key = s.substring(startKey, i++);
StringBuilder buff = new StringBuilder();
while (i < size) {
......@@ -608,4 +609,16 @@ public class DataUtils {
Constants.VERSION_MINOR + "." + Constants.BUILD_ID + "]";
}
static byte[] getPasswordBytes(char[] passwordChars) {
// using UTF-16
int len = passwordChars.length;
byte[] password = new byte[len * 2];
for (int i = 0; i < len; i++) {
char c = passwordChars[i];
password[i + i] = (byte) (c >>> 8);
password[i + i + 1] = (byte) c;
}
return password;
}
}
h2/src/main/org/h2/mvstore/MVStore.java
浏览文件 @ 42af75c5
......@@ -11,6 +11,7 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Collections;
import java.util.Comparator;
......@@ -23,6 +24,7 @@ import org.h2.mvstore.cache.CacheLongKeyLIRS;
import org.h2.mvstore.cache.FilePathCache;
import org.h2.mvstore.type.StringDataType;
import org.h2.store.fs.FilePath;
import org.h2.store.fs.FilePathCrypt2;
import org.h2.store.fs.FileUtils;
import org.h2.util.MathUtils;
import org.h2.util.New;
......@@ -41,8 +43,7 @@ H:3,...
TODO:
- file system encryption: check standard
- file system encryption
- mvcc with multiple transactions
- update checkstyle
- automated 'kill process' and 'power failure' test
......@@ -115,6 +116,7 @@ public class MVStore {
private static final int FORMAT_READ = 1;
private final String fileName;
private final char[] filePassword;
private int pageSize = 6 * 1024;
......@@ -185,8 +187,10 @@ public class MVStore {
int mb = s == null ? 16 : Integer.parseInt(s.toString());
cache = new CacheLongKeyLIRS<Page>(
mb * 1024 * 1024, 2048, 16, mb * 1024 * 1024 / 2048 * 2 / 100);
filePassword = (char[]) config.get("encrypt");
} else {
cache = null;
filePassword = null;
}
}
......@@ -368,12 +372,18 @@ public class MVStore {
return;
}
FileUtils.createDirectories(FileUtils.getParent(fileName));
try {
if (readOnly) {
openFile();
} else if (!openFile()) {
readOnly = true;
openFile();
}
} finally {
if (filePassword != null) {
Arrays.fill(filePassword, (char) 0);
}
}
}
/**
......@@ -391,7 +401,12 @@ public class MVStore {
if (f.exists() && !f.canWrite()) {
readOnly = true;
}
file = FilePathCache.wrap(f.open(readOnly ? "r" : "rw"));
file = f.open(readOnly ? "r" : "rw");
if (filePassword != null) {
byte[] password = DataUtils.getPasswordBytes(filePassword);
file = new FilePathCrypt2.FileCrypt2(password, file);
}
file = FilePathCache.wrap(file);
if (readOnly) {
fileLock = file.tryLock(0, Long.MAX_VALUE, true);
if (fileLock == null) {
......@@ -433,7 +448,7 @@ public class MVStore {
}
} catch (Exception e) {
try {
close();
close(false);
} catch (Exception e2) {
// ignore
}
......@@ -443,6 +458,7 @@ public class MVStore {
return true;
}
private void readMeta() {
Chunk header = readChunkHeader(rootChunkStart);
lastChunkId = header.id;
......@@ -545,10 +561,16 @@ public class MVStore {
* Close the file. Uncommitted changes are ignored, and all open maps are closed.
*/
public void close() {
close(true);
}
private void close(boolean shrinkIfPossible) {
closed = true;
if (file != null) {
try {
if (shrinkIfPossible) {
shrinkFileIfPossible(0);
}
log("file close");
if (fileLock != null) {
fileLock.release();
......@@ -1416,6 +1438,21 @@ public class MVStore {
return set("fileName", fileName);
}
/**
* Encrypt / decrypt the file using the given password. This method has
* no effect for in-memory stores. The password is passed as a char
* array so that it can be cleared as soon as possible. Please note
* there is still a small risk that password stays in memory (due to
* Java garbage collection). Also, the hashed encryption key is kept in
* memory as long as the file is open.
*
* @param password the password
* @return this
*/
public Builder encryptionKey(char[] password) {
return set("encrypt", password);
}
/**
* Open the file in read-only mode. In this case, a shared lock will be
* acquired to ensure the file is not concurrently opened in write mode.
......
h2/src/main/org/h2/security/SHA256.java
浏览文件 @ 42af75c5
......@@ -77,6 +77,54 @@ public class SHA256 {
return getHash(buff, true);
}
public static byte[] getHMAC(byte[] key, byte[] message) {
int blockSize = 64;
if (key.length > blockSize) {
key = getHash(key, false);
}
if (key.length < blockSize) {
key = Arrays.copyOf(key, blockSize);
}
byte[] iKeyPadMessage = new byte[blockSize + message.length];
Arrays.fill(iKeyPadMessage, 0, blockSize, (byte) 0x36);
xor(iKeyPadMessage, key, blockSize);
System.arraycopy(message, 0, iKeyPadMessage, blockSize, message.length);
byte[] k = getHash(iKeyPadMessage, false);
byte[] oKeyPad = new byte[blockSize + k.length];
Arrays.fill(oKeyPad, 0, blockSize, (byte) 0x5c);
xor(oKeyPad, key, blockSize);
System.arraycopy(k, 0, oKeyPad, blockSize, k.length);
return getHash(oKeyPad, false);
}
private static void xor(byte[] target, byte[] data, int len) {
for (int i = 0; i < len; i++) {
target[i] ^= data[i];
}
}
public static byte[] getPBKDF2(byte[] password, byte[] salt, int iterations, int len) {
byte[] result = new byte[len];
byte[] last = null;
for (int k = 1, offset = 0; offset < len; k++, offset += 32) {
for (int i = 0; i < iterations; i++) {
byte[] x;
if (i == 0) {
x = Arrays.copyOf(salt, salt.length + 4);
writeInt(x, salt.length, k);
} else {
x = last;
}
last = getHMAC(password, x);
for (int j = 0; j < 32 && j + offset < len; j++) {
result[j + offset] ^= last[j];
}
}
}
Arrays.fill(password, (byte) 0);
return result;
}
/**
* Calculate the hash code for the given data.
*
......
h2/src/main/org/h2/store/fs/FilePathCrypt2.java 0 → 100644
浏览文件 @ 42af75c5
/*
* Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
* Version 1.0, and under the Eclipse Public License, Version 1.0
* (http://h2database.com/html/license.html).
* Initial Developer: H2 Group
*/
package org.h2.store.fs;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.util.Arrays;
import org.h2.message.DbException;
import org.h2.mvstore.DataUtils;
import org.h2.security.AES;
import org.h2.security.SHA256;
import org.h2.util.MathUtils;
import org.h2.util.StringUtils;
/**
* An encrypted file.
*/
public class FilePathCrypt2 extends FilePathWrapper {
private static final String SCHEME = "crypt2";
/**
* Register this file system.
*/
public static void register() {
FilePath.register(new FilePathCrypt2());
}
public FileChannel open(String mode) throws IOException {
String[] parsed = parse(name);
FileChannel file = FileUtils.open(parsed[1], mode);
byte[] passwordBytes = StringUtils.convertHexToBytes(parsed[0]);
return new FileCrypt2(passwordBytes, file);
}
public String getScheme() {
return SCHEME;
}
protected String getPrefix() {
String[] parsed = parse(name);
return getScheme() + ":" + parsed[0] + ":";
}
public FilePath unwrap(String fileName) {
return FilePath.get(parse(fileName)[1]);
}
public long size() {
long len = getBase().size();
return Math.max(0, len - FileCrypt2.HEADER_LENGTH);
}
public OutputStream newOutputStream(boolean append) {
try {
return new FileChannelOutputStream(open("rw"), append);
} catch (IOException e) {
throw DbException.convertIOException(e, name);
}
}
public InputStream newInputStream() {
try {
return new FileChannelInputStream(open("r"));
} catch (IOException e) {
throw DbException.convertIOException(e, name);
}
}
/**
* Split the file name into algorithm, password, and base file name.
*
* @param fileName the file name
* @return an array with algorithm, password, and base file name
*/
private String[] parse(String fileName) {
if (!fileName.startsWith(getScheme())) {
DbException.throwInternalError(fileName + " doesn't start with " + getScheme());
}
fileName = fileName.substring(getScheme().length() + 1);
int idx = fileName.indexOf(':');
String password;
if (idx < 0) {
DbException.throwInternalError(fileName + " doesn't contain encryption algorithm and password");
}
password = fileName.substring(0, idx);
fileName = fileName.substring(idx + 1);
return new String[] { password, fileName };
}
/**
* An encrypted file with a read cache.
*/
public static class FileCrypt2 extends FileBase {
/**
* The block size.
*/
// TODO use a block size of 2048 and a header size of 4096
static final int BLOCK_SIZE = 4096;
/**
* The length of the file header. Using a smaller header is possible, but
* would mean reads and writes are not aligned to the block size.
*/
static final int HEADER_LENGTH = BLOCK_SIZE;
// TODO improve the header
private static final byte[] HEADER = "H2crypt\n".getBytes();
private static final int SALT_POS = HEADER.length;
/**
* The length of the salt, in bytes.
*/
private static final int SALT_LENGTH = 32;
/**
* The number of iterations.
*/
private static final int HASH_ITERATIONS = 10000;
private final FileChannel base;
private final XTSAES xts;
private long pos;
public FileCrypt2(byte[] passwordBytes, FileChannel base) throws IOException {
// TODO rename 'password' to 'options' (comma separated)
this.base = base;
boolean newFile = base.size() < HEADER_LENGTH;
byte[] salt;
if (newFile) {
byte[] header = Arrays.copyOf(HEADER, BLOCK_SIZE);
salt = MathUtils.secureRandomBytes(SALT_LENGTH);
System.arraycopy(salt, 0, header, SALT_POS, salt.length);
DataUtils.writeFully(base, 0, ByteBuffer.wrap(header));
} else {
salt = new byte[SALT_LENGTH];
DataUtils.readFully(base, SALT_POS, ByteBuffer.wrap(salt));
}
byte[] key = SHA256.getPBKDF2(passwordBytes, salt, HASH_ITERATIONS, 16);
xts = new XTSAES(key);
}
protected void implCloseChannel() throws IOException {
base.close();
}
public FileChannel position(long newPosition) throws IOException {
this.pos = newPosition;
return this;
}
public long position() throws IOException {
return pos;
}
public int read(ByteBuffer dst) throws IOException {
int len = read(dst, pos);
if (len > 0) {
pos += len;
}
return len;
}
public int read(ByteBuffer dst, long position) throws IOException {
int len = dst.remaining();
if (position % BLOCK_SIZE != 0) {
throw new IllegalArgumentException("pos: " + position);
}
if (len % BLOCK_SIZE != 0) {
throw new IllegalArgumentException("len: " + len);
}
int x = dst.position();
len = base.read(dst, position + HEADER_LENGTH);
long block = position / BLOCK_SIZE;
int l = len;
while (l > 0) {
xts.decrypt(block++, BLOCK_SIZE, dst.array(), x);
x += BLOCK_SIZE;
l -= BLOCK_SIZE;
}
return len;
}
public int write(ByteBuffer src, long position) throws IOException {
int len = src.remaining();
// TODO support non-block aligned file length / reads / writes
if (position % BLOCK_SIZE != 0) {
throw new IllegalArgumentException("pos: " + position);
}
if (len % BLOCK_SIZE != 0) {
throw new IllegalArgumentException("len: " + len);
}
ByteBuffer crypt = ByteBuffer.allocate(len);
crypt.put(src);
crypt.flip();
long block = position / BLOCK_SIZE;
int x = 0;
while (len > 0) {
xts.encrypt(block++, BLOCK_SIZE, crypt.array(), x);
x += BLOCK_SIZE;
len -= BLOCK_SIZE;
}
return base.write(crypt, position + BLOCK_SIZE);
}
public int write(ByteBuffer src) throws IOException {
int len = write(src, pos);
if (len > 0) {
pos += len;
}
return len;
}
public long size() throws IOException {
return base.size() - HEADER_LENGTH;
}
public FileChannel truncate(long newSize) throws IOException {
if (newSize % BLOCK_SIZE != 0) {
throw new IllegalArgumentException("newSize: " + newSize);
}
base.truncate(newSize + BLOCK_SIZE);
return this;
}
public void force(boolean metaData) throws IOException {
base.force(metaData);
}
public FileLock tryLock(long position, long size, boolean shared) throws IOException {
return base.tryLock(position, size, shared);
}
public String toString() {
return SCHEME + ":" + base.toString();
}
}
/**
* An XTS-AES implementation as described in
* IEEE P1619 (Standard Architecture for Encrypted Shared Storage Media).
* See also
* http://axelkenzo.ru/downloads/1619-2007-NIST-Submission.pdf
*/
static class XTSAES {
/**
* Galois Field feedback.
*/
private static final int GF_128_FEEDBACK = 0x87;
/**
* The AES encryption block size.
*/
private static final int AES_BLOCK_SIZE = 16;
private final AES aes = new AES();
XTSAES(byte[] key) {
aes.setKey(key);
}
void encrypt(long id, int len, byte[] data, int offset) {
byte[] tweak = initTweak(id);
int i = 0;
for (; i + AES_BLOCK_SIZE <= len; i += AES_BLOCK_SIZE) {
if (i > 0) {
updateTweak(tweak);
}
xorTweak(data, i + offset, tweak);
aes.encrypt(data, i + offset, AES_BLOCK_SIZE);
xorTweak(data, i + offset, tweak);
}
if (i < len) {
updateTweak(tweak);
swap(data, i + offset, i - AES_BLOCK_SIZE + offset, len - i);
xorTweak(data, i - AES_BLOCK_SIZE + offset, tweak);
aes.encrypt(data, i - AES_BLOCK_SIZE + offset, AES_BLOCK_SIZE);
xorTweak(data, i - AES_BLOCK_SIZE + offset, tweak);
}
}
void decrypt(long id, int len, byte[] data, int offset) {
byte[] tweak = initTweak(id), tweakEnd = tweak;
int i = 0;
for (; i + AES_BLOCK_SIZE <= len; i += AES_BLOCK_SIZE) {
if (i > 0) {
updateTweak(tweak);
if (i + AES_BLOCK_SIZE + AES_BLOCK_SIZE > len && i + AES_BLOCK_SIZE < len) {
tweakEnd = Arrays.copyOf(tweak, AES_BLOCK_SIZE);
updateTweak(tweak);
}
}
xorTweak(data, i + offset, tweak);
aes.decrypt(data, i + offset, AES_BLOCK_SIZE);
xorTweak(data, i + offset, tweak);
}
if (i < len) {
swap(data, i, i - AES_BLOCK_SIZE + offset, len - i + offset);
xorTweak(data, i - AES_BLOCK_SIZE + offset, tweakEnd);
aes.decrypt(data, i - AES_BLOCK_SIZE + offset, AES_BLOCK_SIZE);
xorTweak(data, i - AES_BLOCK_SIZE + offset, tweakEnd);
}
}
private byte[] initTweak(long id) {
byte[] tweak = new byte[AES_BLOCK_SIZE];
for (int j = 0; j < AES_BLOCK_SIZE; j++, id >>>= 8) {
tweak[j] = (byte) (id & 0xff);
}
aes.encrypt(tweak, 0, AES_BLOCK_SIZE);
return tweak;
}
private static void xorTweak(byte[] data, int pos, byte[] tweak) {
for (int i = 0; i < AES_BLOCK_SIZE; i++) {
data[pos + i] ^= tweak[i];
}
}
static void updateTweak(byte[] tweak) {
byte ci = 0, co = 0;
for (int i = 0; i < AES_BLOCK_SIZE; i++) {
co = (byte) ((tweak[i] >> 7) & 1);
tweak[i] = (byte) (((tweak[i] << 1) + ci) & 255);
ci = co;
}
if (co != 0) {
tweak[0] ^= GF_128_FEEDBACK;
}
}
static void swap(byte[] data, int source, int target, int len) {
for (int i = 0; i < len; i++) {
byte temp = data[source + i];
data[source + i] = data[target + i];
data[target + i] = temp;
}
}
}
}
h2/src/test/org/h2/test/store/TestMVStore.java
浏览文件 @ 42af75c5
......@@ -40,6 +40,7 @@ public class TestMVStore extends TestBase {
public void test() throws Exception {
FileUtils.deleteRecursive(getBaseDir(), true);
testEncryptedFile();
testFileFormatChange();
testRecreateMap();
testRenameMapRollback();
......@@ -74,6 +75,54 @@ public class TestMVStore extends TestBase {
testSimple();
}
private void testEncryptedFile() {
String fileName = getBaseDir() + "/testEncryptedFile.h3";
FileUtils.delete(fileName);
MVStore s;
MVMap<Integer, String> m;
char[] passwordChars = "007".toCharArray();
s = new MVStore.Builder().
fileName(fileName).
encryptionKey(passwordChars).
open();
assertEquals(0, passwordChars[0]);
assertEquals(0, passwordChars[1]);
assertEquals(0, passwordChars[2]);
assertTrue(FileUtils.exists(fileName));
m = s.openMap("test");
m.put(1, "Hello");
assertEquals("Hello", m.get(1));
s.store();
s.close();
passwordChars = "008".toCharArray();
try {
s = new MVStore.Builder().
fileName(fileName).
encryptionKey(passwordChars).open();
fail();
} catch (IllegalStateException e) {
assertTrue(e.getCause() != null);
}
assertEquals(0, passwordChars[0]);
assertEquals(0, passwordChars[1]);
assertEquals(0, passwordChars[2]);
passwordChars = "007".toCharArray();
s = new MVStore.Builder().
fileName(fileName).
encryptionKey(passwordChars).open();
assertEquals(0, passwordChars[0]);
assertEquals(0, passwordChars[1]);
assertEquals(0, passwordChars[2]);
m = s.openMap("test");
assertEquals("Hello", m.get(1));
s.close();
FileUtils.delete(fileName);
assertFalse(FileUtils.exists(fileName));
}
private void testFileFormatChange() {
String fileName = getBaseDir() + "/testFileFormatChange.h3";
FileUtils.delete(fileName);
......
h2/src/test/org/h2/test/unit/TestSecurity.java
浏览文件 @ 42af75c5
......@@ -46,9 +46,71 @@ public class TestSecurity extends TestBase {
}
private void testSHA() {
testPBKDF2();
testHMAC();
testOneSHA();
}
private void testPBKDF2() {
// test vectors from StackOverflow (PBKDF2-HMAC-SHA2)
assertEquals(
"120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b",
StringUtils.convertBytesToHex(
SHA256.getPBKDF2(
"password".getBytes(),
"salt".getBytes(), 1, 32)));
assertEquals(
"ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43",
StringUtils.convertBytesToHex(
SHA256.getPBKDF2(
"password".getBytes(),
"salt".getBytes(), 2, 32)));
assertEquals(
"c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a",
StringUtils.convertBytesToHex(
SHA256.getPBKDF2(
"password".getBytes(),
"salt".getBytes(), 4096, 32)));
// take a very long time to calculate
// assertEquals(
// "cf81c66fe8cfc04d1f31ecb65dab4089f7f179e89b3b0bcb17ad10e3ac6eba46",
// StringUtils.convertBytesToHex(
// SHA256.getPBKDF2(
// "password".getBytes(),
// "salt".getBytes(), 16777216, 32)));
assertEquals(
"348c89dbcbd32b2f32d814b8116e84cf2b17347ebc1800181c4e2a1fb8dd53e1c635518c7dac47e9",
StringUtils.convertBytesToHex(
SHA256.getPBKDF2(
"passwordPASSWORDpassword".getBytes(),
"saltSALTsaltSALTsaltSALTsaltSALTsalt".getBytes(), 4096, 40)));
assertEquals(
"89b69d0516f829893c696226650a8687",
StringUtils.convertBytesToHex(
SHA256.getPBKDF2(
"pass\0word".getBytes(),
"sa\0lt".getBytes(), 4096, 16)));
// the password is filled with zeroes
byte[] password = "Test".getBytes();
SHA256.getPBKDF2(password, "".getBytes(), 1, 16);
assertEquals(new byte[4], password);
}
private void testHMAC() {
// from Wikipedia
assertEquals(
"b613679a0814d9ec772f95d778c35fc5ff1697c493715653c6c712144292c5ad",
StringUtils.convertBytesToHex(
SHA256.getHMAC(new byte[0], new byte[0])));
assertEquals(
"f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8",
StringUtils.convertBytesToHex(
SHA256.getHMAC(
"key".getBytes(),
"The quick brown fox jumps over the lazy dog".getBytes())));
}
private String getHashString(byte[] data) {
byte[] result = SHA256.getHash(data, true);
if (data.length > 0) {
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论