Skip to content

H
h2database
Unverified 提交 a63f2589 authored 作者: Andrei Tokar's avatar Andrei Tokar 提交者： GitHub
浏览文件
操作

Merge pull request #1344 from h2database/last-chunk-verification 

Streamline last chunk verification on startup
上级 42a81202 1684aacb
隐藏空白字符变更
内嵌 并排
正在显示 包含 81 行增加 和 107 行删除
h2/src/main/org/h2/mvstore/MVStore.java
浏览文件 @ a63f2589
...@@ -706,71 +706,95 @@ public class MVStore { ...@@ -706,71 +706,95 @@ public class MVStore {
} }
} }
} }
if (newest == null) {
// no chunk long blocksInStore = fileStore.size() / BLOCK_SIZE;
return; // this queue will hold potential candidates for lastChunk to fall back to
} Queue<Chunk> lastChunkCandidates = new PriorityQueue<>(Math.max(32, (int)(blocksInStore / 4)), new Comparator<Chunk>() {
// read the chunk header and footer, @Override
// and follow the chain of next chunks public int compare(Chunk one, Chunk two) {
while (true) { int result = Long.compare(two.version, one.version);
if (newest.next == 0 || if (result == 0) {
newest.next >= fileStore.size() / BLOCK_SIZE) { // out of two versions of the same chunk we prefer the one
// no (valid) next // close to the begining of file (presumably later version)
break; result = Long.compare(one.block, two.block);
}
return result;
} }
test = readChunkHeaderAndFooter(newest.next); });
if (test == null || test.id <= newest.id) { Map<Long, Chunk> validChunkCacheByLocation = new HashMap<>();
break;
if (newest != null) {
// read the chunk header and footer,
// and follow the chain of next chunks
while (true) {
validChunkCacheByLocation.put(newest.block, newest);
lastChunkCandidates.add(newest);
if (newest.next == 0 ||
newest.next >= blocksInStore) {
// no (valid) next
break;
}
test = readChunkHeaderAndFooter(newest.next);
if (test == null || test.id <= newest.id) {
break;
}
newest = test;
}
}
// Try candidates for "last chunk" in order from newest to oldest
// until suitable is found. Suitable one should have meta map
// where all chunk references point to valid locations.
boolean verified = false;
while(!verified && setLastChunk(lastChunkCandidates.poll()) != null) {
verified = true;
// load the chunk metadata: although meta's root page resides in the lastChunk,
// traversing meta map might recursively load another chunk(s)
Cursor<String, String> cursor = meta.cursor("chunk.");
while (cursor.hasNext() && cursor.next().startsWith("chunk.")) {
Chunk c = Chunk.fromString(cursor.getValue());
assert c.version <= currentVersion;
// might be there already, due to meta traversal
// see readPage() ... getChunkIfFound()
chunks.putIfAbsent(c.id, c);
long block = c.block;
test = validChunkCacheByLocation.get(block);
if (test == null) {
test = readChunkHeaderAndFooter(block);
if (test != null && test.id == c.id) { // chunk is valid
validChunkCacheByLocation.put(block, test);
lastChunkCandidates.offer(test);
continue;
}
} else if (test.id == c.id) { // chunk is valid
// nothing to do, since chunk was already verified
// and registered as potential "last chunk" candidate
continue;
}
// chunk reference is invalid
// this "last chunk" cadidate is not suitable
// but we continue to process all references
// to find other potential candidates
verified = false;
} }
newest = test;
} }
do {
setLastChunk(newest);
loadChunkMeta();
fileStore.clear();
// build the free space list
for (Chunk c : chunks.values()) {
long start = c.block * BLOCK_SIZE;
int length = c.len * BLOCK_SIZE;
fileStore.markUsed(start, length);
}
assert fileStore.getFileLengthInUse() == measureFileLengthInUse() :
fileStore.getFileLengthInUse() + " != " + measureFileLengthInUse();
// read all chunk headers and footers within the retention time,
// to detect unwritten data after a power failure
} while((newest = verifyLastChunks()) != null);
setWriteVersion(currentVersion); fileStore.clear();
if (lastStoredVersion == INITIAL_VERSION) { // build the free space list
lastStoredVersion = currentVersion - 1; for (Chunk c : chunks.values()) {
long start = c.block * BLOCK_SIZE;
int length = c.len * BLOCK_SIZE;
fileStore.markUsed(start, length);
} }
assert fileStore.getFileLengthInUse() == measureFileLengthInUse() : assert fileStore.getFileLengthInUse() == measureFileLengthInUse() :
fileStore.getFileLengthInUse() + " != " + measureFileLengthInUse(); fileStore.getFileLengthInUse() + " != " + measureFileLengthInUse();
} setWriteVersion(currentVersion);
if (lastStoredVersion == INITIAL_VERSION) {
private void loadChunkMeta() { lastStoredVersion = currentVersion - 1;
// load the chunk metadata: we can load in any order,
// because loading chunk metadata might recursively load another chunk
for (Iterator<String> it = meta.keyIterator("chunk."); it.hasNext();) {
String s = it.next();
if (!s.startsWith("chunk.")) {
break;
}
s = meta.get(s);
Chunk c = Chunk.fromString(s);
if (c.version < lastChunk.version) {
if (chunks.putIfAbsent(c.id, c) == null) {
if (c.block == Long.MAX_VALUE) {
throw DataUtils.newIllegalStateException(
DataUtils.ERROR_FILE_CORRUPT,
"Chunk {0} is invalid", c.id);
}
}
}
} }
} }
private void setLastChunk(Chunk last) { private Chunk setLastChunk(Chunk last) {
chunks.clear(); chunks.clear();
lastChunk = last; lastChunk = last;
if (last == null) { if (last == null) {
...@@ -786,59 +810,9 @@ public class MVStore { ...@@ -786,59 +810,9 @@ public class MVStore {
lastStoredVersion = currentVersion - 1; lastStoredVersion = currentVersion - 1;
meta.setRootPos(last.metaRootPos, lastStoredVersion); meta.setRootPos(last.metaRootPos, lastStoredVersion);
} }
return last;
} }
private Chunk verifyLastChunks() {
assert lastChunk == null || chunks.containsKey(lastChunk.id) : lastChunk;
BitSet validIds = new BitSet();
Queue<Chunk> queue = new PriorityQueue<>(chunks.size(), new Comparator<Chunk>() {
@Override
public int compare(Chunk one, Chunk two) {
return Integer.compare(one.id, two.id);
}
});
queue.addAll(chunks.values());
int newestValidChunk = -1;
Chunk c;
while((c = queue.poll()) != null) {
Chunk test = readChunkHeaderAndFooter(c.block);
if (test == null || test.id != c.id) {
continue;
}
validIds.set(c.id);
try {
MVMap<String, String> oldMeta = meta.openReadOnly(c.metaRootPos, c.version);
boolean valid = true;
for(Iterator<String> iter = oldMeta.keyIterator("chunk."); valid && iter.hasNext(); ) {
String s = iter.next();
if (!s.startsWith("chunk.")) {
break;
}
s = oldMeta.get(s);
valid = validIds.get(Chunk.fromString(s).id);
}
if (valid) {
newestValidChunk = c.id;
}
} catch (Exception ignore) {/**/}
}
Chunk newest = chunks.get(newestValidChunk);
if (newest != lastChunk) {
if (newest == null) {
rollbackTo(0);
} else {
// to avoid re-using newer chunks later on, we could clear
// the headers and footers of those, but we might not know about all
// of them, so that could be incomplete - but we check that newer
// chunks are written after older chunks, so we are safe
rollbackTo(newest.version);
return newest;
}
}
return null;
}
/** /**
* Read a chunk header and footer, and verify the stored data is consistent. * Read a chunk header and footer, and verify the stored data is consistent.
......
h2/src/test/org/h2/test/store/TestMVStore.java
浏览文件 @ a63f2589
...@@ -1451,7 +1451,7 @@ public class TestMVStore extends TestBase { ...@@ -1451,7 +1451,7 @@ public class TestMVStore extends TestBase {
assertEquals(0, m.size()); assertEquals(0, m.size());
s.commit(); s.commit();
// ensure only nodes are read, but not leaves // ensure only nodes are read, but not leaves
assertEquals(10, s.getFileStore().getReadCount()); assertEquals(8, s.getFileStore().getReadCount());
assertTrue(s.getFileStore().getWriteCount() < 5); assertTrue(s.getFileStore().getWriteCount() < 5);
s.close(); s.close();
} }
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论