Skip to content

H
h2database
Unverified 提交 c0a838a1 authored 作者: Noel Grandin's avatar Noel Grandin 提交者： GitHub
浏览文件
操作

Merge pull request #1413 from grandinj/gc_gather_io2 

improvements to MVStore garbage collection
上级 12ec6d9d aeea8c9f
隐藏空白字符变更
内嵌 并排
正在显示 包含 171 行增加 和 130 行删除
h2/src/main/org/h2/mvstore/MVStore.java
浏览文件 @ c0a838a1
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
*/ */
package org.h2.mvstore; package org.h2.mvstore;
import static org.h2.mvstore.MVMap.INITIAL_VERSION;
import java.lang.Thread.UncaughtExceptionHandler; import java.lang.Thread.UncaughtExceptionHandler;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
...@@ -17,11 +18,17 @@ import java.util.HashMap; ...@@ -17,11 +18,17 @@ import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.PriorityQueue; import java.util.PriorityQueue;
import java.util.Queue; import java.util.Queue;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
...@@ -30,9 +37,9 @@ import org.h2.compress.CompressDeflate; ...@@ -30,9 +37,9 @@ import org.h2.compress.CompressDeflate;
import org.h2.compress.CompressLZF; import org.h2.compress.CompressLZF;
import org.h2.compress.Compressor; import org.h2.compress.Compressor;
import org.h2.engine.Constants; import org.h2.engine.Constants;
import org.h2.message.DbException;
import org.h2.mvstore.cache.CacheLongKeyLIRS; import org.h2.mvstore.cache.CacheLongKeyLIRS;
import org.h2.util.MathUtils; import org.h2.util.MathUtils;
import static org.h2.mvstore.MVMap.INITIAL_VERSION;
import org.h2.util.Utils; import org.h2.util.Utils;
/* /*
...@@ -1337,49 +1344,55 @@ public class MVStore { ...@@ -1337,49 +1344,55 @@ public class MVStore {
} }
private Set<Integer> collectReferencedChunks() { private Set<Integer> collectReferencedChunks() {
ChunkIdsCollector collector = new ChunkIdsCollector(meta.getId()); final ThreadPoolExecutor executorService = new ThreadPoolExecutor(10, 10, 10L, TimeUnit.SECONDS,
Set<Long> inspectedRoots = new HashSet<>(); new ArrayBlockingQueue<Runnable>(keysPerPage + 1));
long pos = lastChunk.metaRootPos; final AtomicInteger executingThreadCounter = new AtomicInteger(0);
inspectedRoots.add(pos); try {
collector.visit(pos); ChunkIdsCollector collector = new ChunkIdsCollector(meta.getId());
long oldestVersionToKeep = getOldestVersionToKeep(); Set<Long> inspectedRoots = new HashSet<>();
MVMap.RootReference rootReference = meta.getRoot(); long pos = lastChunk.metaRootPos;
do { inspectedRoots.add(pos);
Page rootPage = rootReference.root; collector.visit(pos, executorService, executingThreadCounter);
pos = rootPage.getPos(); long oldestVersionToKeep = getOldestVersionToKeep();
if (!rootPage.isSaved()) { MVMap.RootReference rootReference = meta.getRoot();
collector.setMapId(meta.getId()); do {
collector.visit(rootPage); Page rootPage = rootReference.root;
} else if(inspectedRoots.add(pos)) { pos = rootPage.getPos();
collector.setMapId(meta.getId()); if (!rootPage.isSaved()) {
collector.visit(pos); collector.setMapId(meta.getId());
} collector.visit(rootPage, executorService, executingThreadCounter);
} else if (inspectedRoots.add(pos)) {
for (Cursor<String, String> c = new Cursor<>(rootPage, "root."); c.hasNext(); ) { collector.setMapId(meta.getId());
String key = c.next(); collector.visit(pos, executorService, executingThreadCounter);
assert key != null;
if (!key.startsWith("root.")) {
break;
} }
pos = DataUtils.parseHexLong(c.getValue());
if (DataUtils.isPageSaved(pos) && inspectedRoots.add(pos)) { for (Cursor<String, String> c = new Cursor<>(rootPage, "root."); c.hasNext();) {
// to allow for something like "root.tmp.123" to be processed String key = c.next();
int mapId = DataUtils.parseHexInt(key.substring(key.lastIndexOf('.') + 1)); assert key != null;
collector.setMapId(mapId); if (!key.startsWith("root.")) {
collector.visit(pos); break;
}
pos = DataUtils.parseHexLong(c.getValue());
if (DataUtils.isPageSaved(pos) && inspectedRoots.add(pos)) {
// to allow for something like "root.tmp.123" to be
// processed
int mapId = DataUtils.parseHexInt(key.substring(key.lastIndexOf('.') + 1));
collector.setMapId(mapId);
collector.visit(pos, executorService, executingThreadCounter);
}
} }
} } while (rootReference.version >= oldestVersionToKeep && (rootReference = rootReference.previous) != null);
} while(rootReference.version >= oldestVersionToKeep && return collector.getReferenced();
(rootReference = rootReference.previous) != null); } finally {
return collector.getReferenced(); executorService.shutdownNow();
}
} }
final class ChunkIdsCollector { final class ChunkIdsCollector {
private final Set<Integer> referenced = new HashSet<>(); /** really a set */
private final ConcurrentHashMap<Integer, Integer> referencedChunks = new ConcurrentHashMap<>();
private final ChunkIdsCollector parent; private final ChunkIdsCollector parent;
private ChunkIdsCollector child;
private int mapId; private int mapId;
ChunkIdsCollector(int mapId) { ChunkIdsCollector(int mapId) {
...@@ -1398,98 +1411,90 @@ public class MVStore { ...@@ -1398,98 +1411,90 @@ public class MVStore {
public void setMapId(int mapId) { public void setMapId(int mapId) {
this.mapId = mapId; this.mapId = mapId;
if (child != null) {
child.setMapId(mapId);
}
} }
public Set<Integer> getReferenced() { public Set<Integer> getReferenced() {
return referenced; Set<Integer> set = new HashSet<>();
set.addAll(referencedChunks.keySet());
return set;
} }
public void visit(Page page) { public void visit(Page page, ThreadPoolExecutor executorService, AtomicInteger executingThreadCounter) {
long pos = page.getPos(); long pos = page.getPos();
if (DataUtils.isPageSaved(pos)) { if (DataUtils.isPageSaved(pos)) {
register(DataUtils.getPageChunkId(pos)); registerChunk(DataUtils.getPageChunkId(pos));
} }
int count = page.map.getChildPageCount(page); int count = page.map.getChildPageCount(page);
if (count > 0) { if (count == 0) {
ChunkIdsCollector childCollector = getChild(); return;
for (int i = 0; i < count; i++) { }
Page childPage = page.getChildPageIfLoaded(i); final ChunkIdsCollector childCollector = new ChunkIdsCollector(this);
if (childPage != null) { for (int i = 0; i < count; i++) {
childCollector.visit(childPage); Page childPage = page.getChildPageIfLoaded(i);
} else { if (childPage != null) {
childCollector.visit(page.getChildPagePos(i)); childCollector.visit(childPage, executorService, executingThreadCounter);
} } else {
} childCollector.visit(page.getChildPagePos(i), executorService, executingThreadCounter);
// and cache resulting set of chunk ids
if (DataUtils.isPageSaved(pos) && cacheChunkRef != null) {
int[] chunkIds = childCollector.getChunkIds();
cacheChunkRef.put(pos, chunkIds, Constants.MEMORY_ARRAY + 4 * chunkIds.length);
} }
} }
// and cache resulting set of chunk ids
if (DataUtils.isPageSaved(pos) && cacheChunkRef != null) {
int[] chunkIds = childCollector.getChunkIds();
cacheChunkRef.put(pos, chunkIds, Constants.MEMORY_ARRAY + 4 * chunkIds.length);
}
} }
public void visit(long pos) { public void visit(long pos, ThreadPoolExecutor executorService, AtomicInteger executingThreadCounter) {
if (!DataUtils.isPageSaved(pos)) { if (!DataUtils.isPageSaved(pos)) {
return; return;
} }
register(DataUtils.getPageChunkId(pos)); registerChunk(DataUtils.getPageChunkId(pos));
if (DataUtils.getPageType(pos) != DataUtils.PAGE_TYPE_LEAF) { if (DataUtils.getPageType(pos) == DataUtils.PAGE_TYPE_LEAF) {
int chunkIds[]; return;
if (cacheChunkRef != null && (chunkIds = cacheChunkRef.get(pos)) != null) { }
// there is a cached set of chunk ids for this position int chunkIds[];
for (int chunkId : chunkIds) { if (cacheChunkRef != null && (chunkIds = cacheChunkRef.get(pos)) != null) {
register(chunkId); // there is a cached set of chunk ids for this position
} for (int chunkId : chunkIds) {
registerChunk(chunkId);
}
} else {
final ChunkIdsCollector childCollector = new ChunkIdsCollector(this);
Page page;
if (cache != null && (page = cache.get(pos)) != null) {
// there is a full page in cache, use it
childCollector.visit(page, executorService, executingThreadCounter);
} else { } else {
ChunkIdsCollector childCollector = getChild(); // page was not cached: read the data
Page page; Chunk chunk = getChunk(pos);
if (cache != null && (page = cache.get(pos)) != null) { long filePos = chunk.block * BLOCK_SIZE;
// there is a full page in cache, use it filePos += DataUtils.getPageOffset(pos);
childCollector.visit(page); if (filePos < 0) {
} else { throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT,
// page was not cached: read the data "Negative position {0}; p={1}, c={2}", filePos, pos, chunk.toString());
Chunk chunk = getChunk(pos);
long filePos = chunk.block * BLOCK_SIZE;
filePos += DataUtils.getPageOffset(pos);
if (filePos < 0) {
throw DataUtils.newIllegalStateException(
DataUtils.ERROR_FILE_CORRUPT,
"Negative position {0}; p={1}, c={2}", filePos, pos, chunk.toString());
}
long maxPos = (chunk.block + chunk.len) * BLOCK_SIZE;
Page.readChildrenPositions(fileStore, pos, filePos, maxPos, childCollector);
}
// and cache resulting set of chunk ids
if (cacheChunkRef != null) {
chunkIds = childCollector.getChunkIds();
cacheChunkRef.put(pos, chunkIds, Constants.MEMORY_ARRAY + 4 * chunkIds.length);
} }
long maxPos = (chunk.block + chunk.len) * BLOCK_SIZE;
Page.readChildrenPositions(fileStore, pos, filePos, maxPos,
childCollector, executorService, executingThreadCounter);
}
// and cache resulting set of chunk ids
if (cacheChunkRef != null) {
chunkIds = childCollector.getChunkIds();
cacheChunkRef.put(pos, chunkIds, Constants.MEMORY_ARRAY + 4 * chunkIds.length);
} }
} }
} }
private ChunkIdsCollector getChild() { private void registerChunk(int chunkId) {
if (child == null) { if (referencedChunks.put(chunkId, 1) == null && parent != null) {
child = new ChunkIdsCollector(this); parent.registerChunk(chunkId);
} else {
child.referenced.clear();
}
return child;
}
private void register(int chunkId) {
if (referenced.add(chunkId) && parent != null) {
parent.register(chunkId);
} }
} }
private int[] getChunkIds() { private int[] getChunkIds() {
int chunkIds[] = new int[referenced.size()]; int chunkIds[] = new int[referencedChunks.size()];
int index = 0; int index = 0;
for (int chunkId : referenced) { for (Integer chunkId : referencedChunks.keySet()) {
chunkIds[index++] = chunkId; chunkIds[index++] = chunkId;
} }
return chunkIds; return chunkIds;
......
h2/src/main/org/h2/mvstore/Page.java
浏览文件 @ c0a838a1
...@@ -5,15 +5,22 @@ ...@@ -5,15 +5,22 @@
*/ */
package org.h2.mvstore; package org.h2.mvstore;
import static org.h2.engine.Constants.MEMORY_ARRAY;
import static org.h2.engine.Constants.MEMORY_OBJECT;
import static org.h2.engine.Constants.MEMORY_POINTER;
import static org.h2.mvstore.DataUtils.PAGE_TYPE_LEAF;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
import org.h2.compress.Compressor; import org.h2.compress.Compressor;
import org.h2.message.DbException;
import org.h2.mvstore.type.DataType; import org.h2.mvstore.type.DataType;
import org.h2.util.Utils; import org.h2.util.Utils;
import static org.h2.engine.Constants.MEMORY_ARRAY;
import static org.h2.engine.Constants.MEMORY_OBJECT;
import static org.h2.engine.Constants.MEMORY_POINTER;
import static org.h2.mvstore.DataUtils.PAGE_TYPE_LEAF;
/** /**
* A page (a node or a leaf). * A page (a node or a leaf).
...@@ -247,9 +254,9 @@ public abstract class Page implements Cloneable ...@@ -247,9 +254,9 @@ public abstract class Page implements Cloneable
* @param maxPos the maximum position (the end of the chunk) * @param maxPos the maximum position (the end of the chunk)
* @param collector to report child pages positions to * @param collector to report child pages positions to
*/ */
static void readChildrenPositions(FileStore fileStore, long pos, static void readChildrenPositions(FileStore fileStore, long pos, long filePos, long maxPos,
long filePos, long maxPos, final MVStore.ChunkIdsCollector collector, final ThreadPoolExecutor executorService,
MVStore.ChunkIdsCollector collector) { final AtomicInteger executingThreadCounter) {
ByteBuffer buff; ByteBuffer buff;
int maxLength = DataUtils.getPageMaxLength(pos); int maxLength = DataUtils.getPageMaxLength(pos);
if (maxLength == DataUtils.PAGE_LARGE) { if (maxLength == DataUtils.PAGE_LARGE) {
...@@ -260,10 +267,8 @@ public abstract class Page implements Cloneable ...@@ -260,10 +267,8 @@ public abstract class Page implements Cloneable
maxLength = (int) Math.min(maxPos - filePos, maxLength); maxLength = (int) Math.min(maxPos - filePos, maxLength);
int length = maxLength; int length = maxLength;
if (length < 0) { if (length < 0) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT,
DataUtils.ERROR_FILE_CORRUPT, "Illegal page length {0} reading at {1}; max pos {2} ", length, filePos, maxPos);
"Illegal page length {0} reading at {1}; max pos {2} ",
length, filePos, maxPos);
} }
buff = fileStore.readFully(filePos, length); buff = fileStore.readFully(filePos, length);
int chunkId = DataUtils.getPageChunkId(pos); int chunkId = DataUtils.getPageChunkId(pos);
...@@ -271,39 +276,70 @@ public abstract class Page implements Cloneable ...@@ -271,39 +276,70 @@ public abstract class Page implements Cloneable
int start = buff.position(); int start = buff.position();
int pageLength = buff.getInt(); int pageLength = buff.getInt();
if (pageLength > maxLength) { if (pageLength > maxLength) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT,
DataUtils.ERROR_FILE_CORRUPT, "File corrupted in chunk {0}, expected page length =< {1}, got {2}", chunkId, maxLength,
"File corrupted in chunk {0}, expected page length =< {1}, got {2}", pageLength);
chunkId, maxLength, pageLength);
} }
buff.limit(start + pageLength); buff.limit(start + pageLength);
short check = buff.getShort(); short check = buff.getShort();
int m = DataUtils.readVarInt(buff); int m = DataUtils.readVarInt(buff);
int mapId = collector.getMapId(); int mapId = collector.getMapId();
if (m != mapId) { if (m != mapId) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT,
DataUtils.ERROR_FILE_CORRUPT, "File corrupted in chunk {0}, expected map id {1}, got {2}", chunkId, mapId, m);
"File corrupted in chunk {0}, expected map id {1}, got {2}",
chunkId, mapId, m);
} }
int checkTest = DataUtils.getCheckValue(chunkId) int checkTest = DataUtils.getCheckValue(chunkId) ^ DataUtils.getCheckValue(offset)
^ DataUtils.getCheckValue(offset)
^ DataUtils.getCheckValue(pageLength); ^ DataUtils.getCheckValue(pageLength);
if (check != (short) checkTest) { if (check != (short) checkTest) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT,
DataUtils.ERROR_FILE_CORRUPT, "File corrupted in chunk {0}, expected check value {1}, got {2}", chunkId, checkTest, check);
"File corrupted in chunk {0}, expected check value {1}, got {2}",
chunkId, checkTest, check);
} }
int len = DataUtils.readVarInt(buff); int len = DataUtils.readVarInt(buff);
int type = buff.get(); int type = buff.get();
if ((type & 1) != DataUtils.PAGE_TYPE_NODE) { if ((type & 1) != DataUtils.PAGE_TYPE_NODE) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT,
DataUtils.ERROR_FILE_CORRUPT,
"Position {0} expected to be a non-leaf", pos); "Position {0} expected to be a non-leaf", pos);
} }
/**
* The logic here is a little awkward. We want to (a) execute reads in parallel, but (b)
* limit the number of threads we create. This is complicated by (a) the algorithm is
* recursive and needs to wait for children before returning up the call-stack, (b) checking
* the size of the thread-pool is not reliable.
*/
final List<Future<?>> futures = new ArrayList<>(len);
for (int i = 0; i <= len; i++) { for (int i = 0; i <= len; i++) {
collector.visit(buff.getLong()); final long childPagePos = buff.getLong();
for (;;) {
int counter = executingThreadCounter.get();
if (counter >= executorService.getMaximumPoolSize()) {
collector.visit(childPagePos, executorService, executingThreadCounter);
break;
} else {
if (executingThreadCounter.compareAndSet(counter, counter + 1)) {
Future<?> f = executorService.submit(new Runnable() {
@Override
public void run() {
try {
collector.visit(childPagePos, executorService, executingThreadCounter);
} finally {
executingThreadCounter.decrementAndGet();
}
}
});
futures.add(f);
break;
}
}
}
}
for (Future<?> f : futures) {
try {
f.get();
} catch (InterruptedException ex) {
throw new RuntimeException(ex);
} catch (ExecutionException ex) {
throw DbException.convert(ex);
}
} }
} }
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论