提交 5f5eb72d authored 作者: Thomas Mueller's avatar Thomas Mueller

MVStore: simpler API

上级 7cb0c93e
...@@ -847,7 +847,7 @@ public class MVMap<K, V> extends AbstractMap<K, V> ...@@ -847,7 +847,7 @@ public class MVMap<K, V> extends AbstractMap<K, V>
* Forget those old versions that are no longer needed. * Forget those old versions that are no longer needed.
*/ */
void removeUnusedOldVersions() { void removeUnusedOldVersions() {
long oldest = store.getRetainOrStoreVersion(); long oldest = store.getOldestVersionToKeep();
if (oldest == -1) { if (oldest == -1) {
return; return;
} }
......
...@@ -57,6 +57,10 @@ MVTableEngine: ...@@ -57,6 +57,10 @@ MVTableEngine:
TransactionStore: TransactionStore:
MVStore: MVStore:
- better document auto-commit (also when many unsaved pages)
advantages and disadvantages
- auto-commit when used in memory as well
- better document MVStore setters
- automated 'kill process' and 'power failure' test - automated 'kill process' and 'power failure' test
- update checkstyle - update checkstyle
- feature to auto-compact from time to time and on close - feature to auto-compact from time to time and on close
...@@ -124,7 +128,7 @@ MVStore: ...@@ -124,7 +128,7 @@ MVStore:
- fix documentation (including examples) - fix documentation (including examples)
- autocommit commits, stores, and compacts from time to time; - autocommit commits, stores, and compacts from time to time;
the background thread should wait at least 90% of the the background thread should wait at least 90% of the
configured write delay to store changes configured write delay to store changes
- currently, uncommitted changes are stored if there are many transient changes, - currently, uncommitted changes are stored if there are many transient changes,
and rolled back when opening - is this really needed? and rolled back when opening - is this really needed?
- compact* should also store uncommitted changes (if there are any) - compact* should also store uncommitted changes (if there are any)
...@@ -177,8 +181,7 @@ public class MVStore { ...@@ -177,8 +181,7 @@ public class MVStore {
/** /**
* The map of chunks. * The map of chunks.
*/ */
private final ConcurrentHashMap<Integer, Chunk> chunks = private final ConcurrentHashMap<Integer, Chunk> chunks = new ConcurrentHashMap<Integer, Chunk>();
new ConcurrentHashMap<Integer, Chunk>();
/** /**
* The map of temporarily freed storage space caused by freed pages. The key * The map of temporarily freed storage space caused by freed pages. The key
...@@ -189,12 +192,12 @@ public class MVStore { ...@@ -189,12 +192,12 @@ public class MVStore {
new ConcurrentHashMap<Long, HashMap<Integer, Chunk>>(); new ConcurrentHashMap<Long, HashMap<Integer, Chunk>>();
/** /**
* The metadata map. Write access to this map needs to be synchronized on the store. * The metadata map. Write access to this map needs to be synchronized on
* the store.
*/ */
private MVMap<String, String> meta; private MVMapConcurrent<String, String> meta;
private final ConcurrentHashMap<Integer, MVMap<?, ?>> maps = private final ConcurrentHashMap<Integer, MVMap<?, ?>> maps = new ConcurrentHashMap<Integer, MVMap<?, ?>>();
new ConcurrentHashMap<Integer, MVMap<?, ?>>();
private HashMap<String, String> storeHeader = New.hashMap(); private HashMap<String, String> storeHeader = New.hashMap();
...@@ -202,7 +205,7 @@ public class MVStore { ...@@ -202,7 +205,7 @@ public class MVStore {
private int lastMapId; private int lastMapId;
private long retainVersion = -1; private int versionsToKeep = 5;
/** /**
* Whether to compress new pages. Even if disabled, the store may contain * Whether to compress new pages. Even if disabled, the store may contain
...@@ -222,14 +225,13 @@ public class MVStore { ...@@ -222,14 +225,13 @@ public class MVStore {
private long lastStoredVersion; private long lastStoredVersion;
/** /**
* The estimated number of unsaved pages * The estimated number of unsaved pages (this number may not be completely
* (this number may not be completely accurate, * accurate, because it may be changed concurrently, and because temporary
* because it may be changed concurrently, and * pages are counted)
* because temporary pages are counted)
*/ */
private int unsavedPageCount; private int unsavedPageCount;
private int unsavedPageCountMax; private int autoCommitPageCount;
private boolean storeNeeded; private boolean saveNeeded;
/** /**
* The time the store was created, in milliseconds since 1970. * The time the store was created, in milliseconds since 1970.
...@@ -237,7 +239,7 @@ public class MVStore { ...@@ -237,7 +239,7 @@ public class MVStore {
private long creationTime; private long creationTime;
private int retentionTime; private int retentionTime;
private long lastStoreTime; private long lastCommitTime;
/** /**
* The earliest chunk to retain, if any. * The earliest chunk to retain, if any.
...@@ -248,15 +250,15 @@ public class MVStore { ...@@ -248,15 +250,15 @@ public class MVStore {
* The version of the current store operation (if any). * The version of the current store operation (if any).
*/ */
private volatile long currentStoreVersion = -1; private volatile long currentStoreVersion = -1;
private Thread currentStoreThread; private Thread currentStoreThread;
private volatile boolean metaChanged; private volatile boolean metaChanged;
/** /**
* The delay in milliseconds to automatically store changes. * The delay in milliseconds to automatically commit and write changes.
*/ */
private int writeDelay; private int autoCommitDelay;
/** /**
* Create and open the store. * Create and open the store.
...@@ -272,7 +274,8 @@ public class MVStore { ...@@ -272,7 +274,8 @@ public class MVStore {
pageSplitSize = o == null ? 6 * 1024 : (Integer) o; pageSplitSize = o == null ? 6 * 1024 : (Integer) o;
o = config.get("backgroundExceptionHandler"); o = config.get("backgroundExceptionHandler");
this.backgroundExceptionHandler = (UncaughtExceptionHandler) o; this.backgroundExceptionHandler = (UncaughtExceptionHandler) o;
meta = new MVMap<String, String>(StringDataType.INSTANCE, StringDataType.INSTANCE); meta = new MVMapConcurrent<String, String>(StringDataType.INSTANCE,
StringDataType.INSTANCE);
HashMap<String, String> c = New.hashMap(); HashMap<String, String> c = New.hashMap();
c.put("id", "0"); c.put("id", "0");
c.put("createVersion", Long.toString(currentVersion)); c.put("createVersion", Long.toString(currentVersion));
...@@ -295,21 +298,21 @@ public class MVStore { ...@@ -295,21 +298,21 @@ public class MVStore {
int averageMemory = Math.max(10, pageSplitSize / 2); int averageMemory = Math.max(10, pageSplitSize / 2);
int segmentCount = 16; int segmentCount = 16;
int stackMoveDistance = maxMemoryBytes / averageMemory * 2 / 100; int stackMoveDistance = maxMemoryBytes / averageMemory * 2 / 100;
cache = new CacheLongKeyLIRS<Page>( cache = new CacheLongKeyLIRS<Page>(maxMemoryBytes, averageMemory,
maxMemoryBytes, averageMemory, segmentCount, stackMoveDistance); segmentCount, stackMoveDistance);
} }
o = config.get("writeBufferSize"); o = config.get("autoCommitBufferSize");
mb = o == null ? 4 : (Integer) o; mb = o == null ? 4 : (Integer) o;
int writeBufferSize = mb * 1024 * 1024; int autoCommitBufferSize = mb * 1024 * 1024;
int div = pageSplitSize; int div = pageSplitSize;
unsavedPageCountMax = writeBufferSize / (div == 0 ? 1 : div); autoCommitPageCount = autoCommitBufferSize / (div == 0 ? 1 : div);
char[] encryptionKey = (char[]) config.get("encryptionKey"); char[] encryptionKey = (char[]) config.get("encryptionKey");
try { try {
fileStore.open(fileName, readOnly, encryptionKey); fileStore.open(fileName, readOnly, encryptionKey);
if (fileStore.size() == 0) { if (fileStore.size() == 0) {
creationTime = 0; creationTime = 0;
creationTime = getTime(); creationTime = getTime();
lastStoreTime = creationTime; lastCommitTime = creationTime;
storeHeader.put("H", "3"); storeHeader.put("H", "3");
storeHeader.put("blockSize", "" + BLOCK_SIZE); storeHeader.put("blockSize", "" + BLOCK_SIZE);
storeHeader.put("format", "" + FORMAT_WRITE); storeHeader.put("format", "" + FORMAT_WRITE);
...@@ -348,14 +351,13 @@ public class MVStore { ...@@ -348,14 +351,13 @@ public class MVStore {
Arrays.fill(encryptionKey, (char) 0); Arrays.fill(encryptionKey, (char) 0);
} }
} }
lastStoreTime = getTime(); lastCommitTime = getTime();
// setWriteDelay starts the thread, but only if
// the parameter is different than the current value
o = config.get("writeDelay"); // setAutoCommitDelay starts the thread, but only if
int writeDelay = o == null ? 1000 : (Integer) o; // the parameter is different from the old value
setWriteDelay(writeDelay); o = config.get("autoCommitDelay");
int delay = o == null ? 1000 : (Integer) o;
setAutoCommitDelay(delay);
} }
/** /**
...@@ -380,7 +382,8 @@ public class MVStore { ...@@ -380,7 +382,8 @@ public class MVStore {
* @return the read-only map * @return the read-only map
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
<T extends MVMap<?, ?>> T openMapVersion(long version, int mapId, MVMap<?, ?> template) { <T extends MVMap<?, ?>> T openMapVersion(long version, int mapId,
MVMap<?, ?> template) {
MVMap<String, String> oldMeta = getMetaMap(version); MVMap<String, String> oldMeta = getMetaMap(version);
String r = oldMeta.get("root." + mapId); String r = oldMeta.get("root." + mapId);
long rootPos = DataUtils.parseLong(r, 0); long rootPos = DataUtils.parseLong(r, 0);
...@@ -468,7 +471,7 @@ public class MVStore { ...@@ -468,7 +471,7 @@ public class MVStore {
* root.{mapId} = {root position} * root.{mapId} = {root position}
* setting.storeVersion = {version} * setting.storeVersion = {version}
* </pre> * </pre>
* *
* @return the metadata map * @return the metadata map
*/ */
public MVMap<String, String> getMetaMap() { public MVMap<String, String> getMetaMap() {
...@@ -512,7 +515,7 @@ public class MVStore { ...@@ -512,7 +515,7 @@ public class MVStore {
metaChanged = true; metaChanged = true;
} }
private void readMeta() { private synchronized void readMeta() {
chunks.clear(); chunks.clear();
Chunk header = readChunkHeader(rootChunkStart); Chunk header = readChunkHeader(rootChunkStart);
if (header.start == Long.MAX_VALUE) { if (header.start == Long.MAX_VALUE) {
...@@ -569,7 +572,8 @@ public class MVStore { ...@@ -569,7 +572,8 @@ public class MVStore {
// we don't know which chunk is the newest // we don't know which chunk is the newest
long newestChunk = -1; long newestChunk = -1;
// read the last block of the file, and then the two first blocks // read the last block of the file, and then the two first blocks
ByteBuffer buffLastBlock = fileStore.readFully(fileStore.size() - BLOCK_SIZE, BLOCK_SIZE); ByteBuffer buffLastBlock = fileStore.readFully(fileStore.size()
- BLOCK_SIZE, BLOCK_SIZE);
ByteBuffer buffFirst2Blocks = fileStore.readFully(0, BLOCK_SIZE * 2); ByteBuffer buffFirst2Blocks = fileStore.readFully(0, BLOCK_SIZE * 2);
ByteBuffer buff = ByteBuffer.allocate(3 * BLOCK_SIZE); ByteBuffer buff = ByteBuffer.allocate(3 * BLOCK_SIZE);
buff.put(buffLastBlock); buff.put(buffLastBlock);
...@@ -611,7 +615,8 @@ public class MVStore { ...@@ -611,7 +615,8 @@ public class MVStore {
} }
if (currentVersion < 0) { if (currentVersion < 0) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(
DataUtils.ERROR_FILE_CORRUPT, "Store header is corrupt: {0}", fileStore); DataUtils.ERROR_FILE_CORRUPT,
"Store header is corrupt: {0}", fileStore);
} }
setWriteVersion(currentVersion); setWriteVersion(currentVersion);
lastStoredVersion = -1; lastStoredVersion = -1;
...@@ -647,10 +652,7 @@ public class MVStore { ...@@ -647,10 +652,7 @@ public class MVStore {
} }
/** /**
* Close the file and the store. If there are any uncommitted changes, they * Close the file and the store. Unsaved changes are written to disk first.
* are written to disk first. All open maps are closed.
* <p>
* It is not allowed to concurrently call close and store.
*/ */
public void close() { public void close() {
if (closed) { if (closed) {
...@@ -659,7 +661,7 @@ public class MVStore { ...@@ -659,7 +661,7 @@ public class MVStore {
if (fileStore != null && !fileStore.isReadOnly()) { if (fileStore != null && !fileStore.isReadOnly()) {
stopBackgroundThread(); stopBackgroundThread();
if (hasUnsavedChanges()) { if (hasUnsavedChanges()) {
store(); commitAndSave();
} }
} }
closeStore(true); closeStore(true);
...@@ -722,6 +724,11 @@ public class MVStore { ...@@ -722,6 +724,11 @@ public class MVStore {
int chunkId = DataUtils.getPageChunkId(pos); int chunkId = DataUtils.getPageChunkId(pos);
Chunk c = chunks.get(chunkId); Chunk c = chunks.get(chunkId);
if (c == null) { if (c == null) {
if (!Thread.holdsLock(this)) {
throw DataUtils.newIllegalStateException(
DataUtils.ERROR_INTERNAL,
"Unsynchronized metadata read");
}
String s = meta.get("chunk." + chunkId); String s = meta.get("chunk." + chunkId);
if (s == null) { if (s == null) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(
...@@ -747,18 +754,22 @@ public class MVStore { ...@@ -747,18 +754,22 @@ public class MVStore {
} }
/** /**
* Commit the changes. This method marks the changes as committed and * Commit the changes.
* increments the version.
* <p> * <p>
* Unless the write delay is set to 0, this method does not write to the * For in-memory stores, this method increments the version.
* file. Instead, data is written after the delay, manually by calling the * <p>
* store method, when the write buffer is full, or when closing the store. * For persistent stores, it also writes changes to disk. It does nothing if
* there are no unsaved changes, and returns the old version. It is not
* necessary to call this method when auto-commit is enabled (the default
* setting), as in this case it is automatically called from time to time or
* when enough changes have accumulated. However, it may still be called to
* flush all changes to disk.
* *
* @return the new version * @return the new version
*/ */
public long commit() { public long commit() {
if (fileStore != null) { if (fileStore != null) {
return store(); return commitAndSave();
} }
long v = ++currentVersion; long v = ++currentVersion;
setWriteVersion(v); setWriteVersion(v);
...@@ -774,13 +785,14 @@ public class MVStore { ...@@ -774,13 +785,14 @@ public class MVStore {
* *
* @return the new version (incremented if there were changes) * @return the new version (incremented if there were changes)
*/ */
public synchronized long store() { private synchronized long commitAndSave() {
if (closed) { if (closed) {
return currentVersion; return currentVersion;
} }
if (fileStore == null) { if (fileStore == null) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(
DataUtils.ERROR_WRITING_FAILED, "This is an in-memory store"); DataUtils.ERROR_WRITING_FAILED,
"This is an in-memory store");
} }
if (currentStoreVersion >= 0) { if (currentStoreVersion >= 0) {
// store is possibly called within store, if the meta map changed // store is possibly called within store, if the meta map changed
...@@ -811,7 +823,7 @@ public class MVStore { ...@@ -811,7 +823,7 @@ public class MVStore {
long version = ++currentVersion; long version = ++currentVersion;
setWriteVersion(version); setWriteVersion(version);
long time = getTime(); long time = getTime();
lastStoreTime = time; lastCommitTime = time;
retainChunk = null; retainChunk = null;
// the last chunk was not completely correct in the last store() // the last chunk was not completely correct in the last store()
...@@ -903,14 +915,6 @@ public class MVStore { ...@@ -903,14 +915,6 @@ public class MVStore {
} }
boolean storeAtEndOfFile = filePos + length >= end; boolean storeAtEndOfFile = filePos + length >= end;
// try {
// Thread.sleep(10);
// } catch (InterruptedException e) {
// ; int todo;
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
c.start = filePos; c.start = filePos;
c.length = chunkLength; c.length = chunkLength;
c.metaRootPos = metaRoot.getPos(); c.metaRootPos = metaRoot.getPos();
...@@ -944,8 +948,10 @@ public class MVStore { ...@@ -944,8 +948,10 @@ public class MVStore {
} }
metaRoot.writeEnd(); metaRoot.writeEnd();
// some pages might have been changed in the meantime (in the newest version) // some pages might have been changed in the meantime (in the newest
unsavedPageCount = Math.max(0, unsavedPageCount - currentUnsavedPageCount); // version)
unsavedPageCount = Math.max(0, unsavedPageCount
- currentUnsavedPageCount);
metaChanged = false; metaChanged = false;
lastStoredVersion = storeVersion; lastStoredVersion = storeVersion;
...@@ -1098,7 +1104,8 @@ public class MVStore { ...@@ -1098,7 +1104,8 @@ public class MVStore {
long size = 2 * BLOCK_SIZE; long size = 2 * BLOCK_SIZE;
for (Chunk c : chunks.values()) { for (Chunk c : chunks.values()) {
long x = c.start + c.length; long x = c.start + c.length;
size = Math.max(size, MathUtils.roundUpLong(x, BLOCK_SIZE) + BLOCK_SIZE); size = Math.max(size, MathUtils.roundUpLong(x, BLOCK_SIZE)
+ BLOCK_SIZE);
} }
return size; return size;
} }
...@@ -1109,8 +1116,6 @@ public class MVStore { ...@@ -1109,8 +1116,6 @@ public class MVStore {
* @return if there are any changes * @return if there are any changes
*/ */
public boolean hasUnsavedChanges() { public boolean hasUnsavedChanges() {
// TODO maybe private; rename to hasUncommittedChanges
checkOpen(); checkOpen();
if (metaChanged) { if (metaChanged) {
return true; return true;
...@@ -1160,7 +1165,8 @@ public class MVStore { ...@@ -1160,7 +1165,8 @@ public class MVStore {
chunks.remove(c.id); chunks.remove(c.id);
markMetaChanged(); markMetaChanged();
meta.remove("chunk." + c.id); meta.remove("chunk." + c.id);
int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE; int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE)
+ BLOCK_SIZE;
fileStore.free(c.start, length); fileStore.free(c.start, length);
} }
if (fileStore.getFillRate() == 100) { if (fileStore.getFillRate() == 100) {
...@@ -1175,7 +1181,8 @@ public class MVStore { ...@@ -1175,7 +1181,8 @@ public class MVStore {
} }
for (Chunk c : move) { for (Chunk c : move) {
WriteBuffer buff = getWriteBuffer(); WriteBuffer buff = getWriteBuffer();
int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE; int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE)
+ BLOCK_SIZE;
buff.limit(length); buff.limit(length);
ByteBuffer buff2 = fileStore.readFully(c.start, length); ByteBuffer buff2 = fileStore.readFully(c.start, length);
buff.put(buff2); buff.put(buff2);
...@@ -1200,7 +1207,7 @@ public class MVStore { ...@@ -1200,7 +1207,7 @@ public class MVStore {
// update the metadata (store at the end of the file) // update the metadata (store at the end of the file)
reuseSpace = false; reuseSpace = false;
store(); commitAndSave();
sync(); sync();
...@@ -1213,7 +1220,8 @@ public class MVStore { ...@@ -1213,7 +1220,8 @@ public class MVStore {
continue; continue;
} }
WriteBuffer buff = getWriteBuffer(); WriteBuffer buff = getWriteBuffer();
int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE) + BLOCK_SIZE; int length = MathUtils.roundUpInt(c.length, BLOCK_SIZE)
+ BLOCK_SIZE;
buff.limit(length); buff.limit(length);
ByteBuffer buff2 = fileStore.readFully(c.start, length); ByteBuffer buff2 = fileStore.readFully(c.start, length);
buff.put(buff2); buff.put(buff2);
...@@ -1235,7 +1243,7 @@ public class MVStore { ...@@ -1235,7 +1243,7 @@ public class MVStore {
} }
// update the metadata (within the file) // update the metadata (within the file)
store(); commitAndSave();
sync(); sync();
shrinkFileIfPossible(0); shrinkFileIfPossible(0);
...@@ -1309,7 +1317,8 @@ public class MVStore { ...@@ -1309,7 +1317,8 @@ public class MVStore {
Collections.sort(old, new Comparator<Chunk>() { Collections.sort(old, new Comparator<Chunk>() {
@Override @Override
public int compare(Chunk o1, Chunk o2) { public int compare(Chunk o1, Chunk o2) {
return new Integer(o1.collectPriority).compareTo(o2.collectPriority); return new Integer(o1.collectPriority)
.compareTo(o2.collectPriority);
} }
}); });
...@@ -1341,7 +1350,7 @@ public class MVStore { ...@@ -1341,7 +1350,7 @@ public class MVStore {
copyLive(c, old); copyLive(c, old);
} }
store(); commitAndSave();
return true; return true;
} }
...@@ -1413,7 +1422,8 @@ public class MVStore { ...@@ -1413,7 +1422,8 @@ public class MVStore {
filePos += DataUtils.getPageOffset(pos); filePos += DataUtils.getPageOffset(pos);
if (filePos < 0) { if (filePos < 0) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(
DataUtils.ERROR_FILE_CORRUPT, "Negative position {0}", filePos); DataUtils.ERROR_FILE_CORRUPT,
"Negative position {0}", filePos);
} }
p = Page.read(fileStore, map, pos, filePos, fileStore.size()); p = Page.read(fileStore, map, pos, filePos, fileStore.size());
if (cache != null) { if (cache != null) {
...@@ -1460,11 +1470,13 @@ public class MVStore { ...@@ -1460,11 +1470,13 @@ public class MVStore {
registerFreePage(version, c.id, DataUtils.getPageMaxLength(pos), 1); registerFreePage(version, c.id, DataUtils.getPageMaxLength(pos), 1);
} }
private void registerFreePage(long version, int chunkId, long maxLengthLive, int pageCount) { private void registerFreePage(long version, int chunkId,
long maxLengthLive, int pageCount) {
HashMap<Integer, Chunk> freed = freedPageSpace.get(version); HashMap<Integer, Chunk> freed = freedPageSpace.get(version);
if (freed == null) { if (freed == null) {
freed = New.hashMap(); freed = New.hashMap();
HashMap<Integer, Chunk> f2 = freedPageSpace.putIfAbsent(version, freed); HashMap<Integer, Chunk> f2 = freedPageSpace.putIfAbsent(version,
freed);
if (f2 != null) { if (f2 != null) {
freed = f2; freed = f2;
} }
...@@ -1541,13 +1553,13 @@ public class MVStore { ...@@ -1541,13 +1553,13 @@ public class MVStore {
} }
/** /**
* Which version to retain in memory. If not set, all versions back to the * How many versions to retain for in-memory stores. If not set, 5 versions
* last stored version are retained. * are retained.
* *
* @param retainVersion the oldest version to retain * @param count the number of versions to keep
*/ */
public void setRetainVersion(long retainVersion) { public void setVersionsToKeep(int count) {
this.retainVersion = retainVersion; this.versionsToKeep = count;
} }
/** /**
...@@ -1555,13 +1567,8 @@ public class MVStore { ...@@ -1555,13 +1567,8 @@ public class MVStore {
* *
* @return the version * @return the version
*/ */
public long getRetainVersion() { public long getVersionsToKeep() {
long v = retainVersion; return versionsToKeep;
long storeVersion = currentStoreVersion;
if (storeVersion > -1) {
v = Math.min(v, storeVersion);
}
return v;
} }
/** /**
...@@ -1570,8 +1577,11 @@ public class MVStore { ...@@ -1570,8 +1577,11 @@ public class MVStore {
* *
* @return the version * @return the version
*/ */
long getRetainOrStoreVersion() { long getOldestVersionToKeep() {
long v = retainVersion; long v = currentVersion;
if (fileStore == null) {
return v - versionsToKeep;
}
long storeVersion = currentStoreVersion; long storeVersion = currentStoreVersion;
if (storeVersion > -1) { if (storeVersion > -1) {
v = Math.min(v, storeVersion); v = Math.min(v, storeVersion);
...@@ -1618,34 +1628,13 @@ public class MVStore { ...@@ -1618,34 +1628,13 @@ public class MVStore {
return true; return true;
} }
/**
* Get the estimated number of unsaved pages. The returned value is not
* accurate, specially after rollbacks, but can be used to estimate the
* memory usage for unsaved data.
*
* @return the number of unsaved pages
*/
public int getUnsavedPageCount() {
return unsavedPageCount;
}
/**
* Get the maximum number of unsaved pages. If this number is exceeded, the
* unsaved changes are stored to disk.
*
* @return the number of maximum unsaved pages
*/
public int getUnsavedPageCountMax() {
return unsavedPageCountMax;
}
/** /**
* Increment the number of unsaved pages. * Increment the number of unsaved pages.
*/ */
void registerUnsavedPage() { void registerUnsavedPage() {
int count = ++unsavedPageCount; int count = ++unsavedPageCount;
if (count > unsavedPageCountMax && unsavedPageCountMax > 0) { if (count > autoCommitPageCount && autoCommitPageCount > 0) {
storeNeeded = true; saveNeeded = true;
} }
} }
...@@ -1653,9 +1642,9 @@ public class MVStore { ...@@ -1653,9 +1642,9 @@ public class MVStore {
* This method is called before writing to a map. * This method is called before writing to a map.
*/ */
void beforeWrite() { void beforeWrite() {
if (storeNeeded) { if (saveNeeded) {
storeNeeded = false; saveNeeded = false;
store(); commitAndSave();
} }
} }
...@@ -1682,13 +1671,12 @@ public class MVStore { ...@@ -1682,13 +1671,12 @@ public class MVStore {
markMetaChanged(); markMetaChanged();
meta.put("setting.storeVersion", Integer.toString(version)); meta.put("setting.storeVersion", Integer.toString(version));
} }
/** /**
* Revert to the beginning of the current version. * Revert to the beginning of the current version, reverting all uncommitted
* changes.
*/ */
public void rollback() { public void rollback() {
// TODO document and test
rollbackTo(currentVersion); rollbackTo(currentVersion);
} }
...@@ -1841,11 +1829,11 @@ public class MVStore { ...@@ -1841,11 +1829,11 @@ public class MVStore {
private void checkOpen() { private void checkOpen() {
if (closed) { if (closed) {
throw DataUtils.newIllegalStateException( throw DataUtils.newIllegalStateException(DataUtils.ERROR_CLOSED,
DataUtils.ERROR_CLOSED, "This store is closed"); "This store is closed");
} }
} }
/** /**
* Rename a map. * Rename a map.
* *
...@@ -1901,25 +1889,25 @@ public class MVStore { ...@@ -1901,25 +1889,25 @@ public class MVStore {
} }
/** /**
* Store all unsaved changes, if there are any that are committed. * Commit and save all changes, if there are any.
*/ */
void storeInBackground() { void commitInBackground() {
if (unsavedPageCount == 0 || closed) { if (unsavedPageCount == 0 || closed) {
return; return;
} }
// could also store when there are many unsaved pages, // could also store when there are many unsaved pages,
// but according to a test it doesn't really help // but according to a test it doesn't really help
long time = getTime(); long time = getTime();
if (time <= lastStoreTime + writeDelay) { if (time <= lastCommitTime + autoCommitDelay) {
return; return;
} }
if (!hasUnsavedChanges()) { if (!hasUnsavedChanges()) {
return; return;
} }
try { try {
store(); commitAndSave();
} catch (Exception e) { } catch (Exception e) {
if (backgroundExceptionHandler != null) { if (backgroundExceptionHandler != null) {
backgroundExceptionHandler.uncaughtException(null, e); backgroundExceptionHandler.uncaughtException(null, e);
...@@ -1959,25 +1947,21 @@ public class MVStore { ...@@ -1959,25 +1947,21 @@ public class MVStore {
} }
/** /**
* Set the maximum delay in milliseconds to commit changes. * Set the maximum delay in milliseconds to auto-commit changes.
* <p> * <p>
* The default is 1000, meaning changes are committed after at * To disable auto-commit, set the value to 0. In this case, changes are
* most one second. * only committed when explicitly calling commit.
* <p> * <p>
* When the value is set to -1, changes are only written when * The default is 1000, meaning all changes are committed after at most one
* calling the store method. When the value is set to 0, committed * second.
* changes are immediately written on a commit, but please note this
* decreases performance and does still not guarantee the disk will
* actually write the data.
* *
* @param millis the maximum delay * @param millis the maximum delay
*/ */
public void setWriteDelay(int millis) { public void setAutoCommitDelay(int millis) {
// TODO rename to commitDelay if (autoCommitDelay == millis) {
if (writeDelay == millis) {
return; return;
} }
writeDelay = millis; autoCommitDelay = millis;
if (fileStore == null) { if (fileStore == null) {
return; return;
} }
...@@ -1991,8 +1975,36 @@ public class MVStore { ...@@ -1991,8 +1975,36 @@ public class MVStore {
} }
} }
public int getWriteDelay() { /**
return writeDelay; * Get the auto-commit delay.
*
* @return the delay in milliseconds, or 0 if auto-commit is disabled.
*/
public int getAutoCommitDelay() {
return autoCommitDelay;
}
/**
* Get the maximum number of unsaved pages. If this number is exceeded,
* unsaved changes are stored to disk.
*
* @return the number of maximum unsaved pages
*/
public int getAutoCommitPageCount() {
return autoCommitPageCount;
}
/**
* Get the estimated number of unsaved pages. If the value exceeds the
* auto-commit page count, the changes are committed.
* <p>
* The returned value may not be completely accurate, but can be used to
* estimate the memory usage for unsaved data.
*
* @return the number of unsaved pages
*/
public int getUnsavedPageCount() {
return unsavedPageCount;
} }
/** /**
...@@ -2000,9 +2012,9 @@ public class MVStore { ...@@ -2000,9 +2012,9 @@ public class MVStore {
*/ */
private static class BackgroundWriterThread extends Thread { private static class BackgroundWriterThread extends Thread {
public final Object sync = new Object();
private final MVStore store; private final MVStore store;
private final int sleep; private final int sleep;
public final Object sync = new Object();
BackgroundWriterThread(MVStore store, int sleep, String fileStoreName) { BackgroundWriterThread(MVStore store, int sleep, String fileStoreName) {
super("MVStore background writer " + fileStoreName); super("MVStore background writer " + fileStoreName);
...@@ -2025,7 +2037,7 @@ public class MVStore { ...@@ -2025,7 +2037,7 @@ public class MVStore {
continue; continue;
} }
} }
store.storeInBackground(); store.commitInBackground();
} }
} }
...@@ -2043,6 +2055,38 @@ public class MVStore { ...@@ -2043,6 +2055,38 @@ public class MVStore {
return this; return this;
} }
/**
* Disable auto-commit, by setting the auto-commit delay and auto-commit
* buffer size to 0.
*
* @return this
*/
public Builder autoCommitDisabled() {
// we have a separate config option so that
// no thread is started if the write delay is 0
// (if we only had a setter in the MVStore,
// the thread would need to be started in any case)
set("autoCommitBufferSize", 0);
return set("autoCommitDelay", 0);
}
/**
* Set the size of the write buffer, in MB (for file-based stores).
* Unless auto-commit is disabled, changes are automatically saved if
* there are more than this amount of changes.
* <p>
* The default is 4 MB.
* <p>
* When the value is set to 0 or lower, data is not automatically
* stored.
*
* @param mb the write buffer size, in megabytes
* @return this
*/
public Builder autoCommitBufferSize(int mb) {
return set("autoCommitBufferSize", mb);
}
/** /**
* Use the following file name. If the file does not exist, it is * Use the following file name. If the file does not exist, it is
* automatically created. The parent directory already must exist. * automatically created. The parent directory already must exist.
...@@ -2076,8 +2120,8 @@ public class MVStore { ...@@ -2076,8 +2120,8 @@ public class MVStore {
* If this option is not used, the file is locked exclusively. * If this option is not used, the file is locked exclusively.
* <p> * <p>
* Please note a store may only be opened once in every JVM (no matter * Please note a store may only be opened once in every JVM (no matter
* whether it is opened in read-only or read-write mode), because each file * whether it is opened in read-only or read-write mode), because each
* may be locked only once in a process. * file may be locked only once in a process.
* *
* @return this * @return this
*/ */
...@@ -2110,23 +2154,6 @@ public class MVStore { ...@@ -2110,23 +2154,6 @@ public class MVStore {
return set("compress", 1); return set("compress", 1);
} }
/**
* Set the size of the write buffer, in MB (for file-based stores).
* Unless auto-commit is disabled, changes are automatically stored if
* the buffer grows larger than this.
* <p>
* The default is 4 MB.
* <p>
* When the value is set to 0 or lower, data is never automatically
* stored.
*
* @param mb the write buffer size, in megabytes
* @return this
*/
public Builder writeBufferSize(int mb) {
return set("writeBufferSize", mb);
}
/** /**
* Set the amount of memory a page should contain at most, in bytes, * Set the amount of memory a page should contain at most, in bytes,
* before it is split. The default is 6 KB. This is not a limit in the * before it is split. The default is 6 KB. This is not a limit in the
...@@ -2141,8 +2168,8 @@ public class MVStore { ...@@ -2141,8 +2168,8 @@ public class MVStore {
} }
/** /**
* Set the listener to be used for exceptions that occur in the background * Set the listener to be used for exceptions that occur when writing in
* thread. * the background thread.
* *
* @param exceptionHandler the handler * @param exceptionHandler the handler
* @return this * @return this
...@@ -2162,20 +2189,6 @@ public class MVStore { ...@@ -2162,20 +2189,6 @@ public class MVStore {
return set("fileStore", store); return set("fileStore", store);
} }
/**
* Set the initial write delay.
*
* @param writeDelay the write delay
* @return this
*/
public Builder writeDelay(int writeDelay) {
// we have a separate config option so that
// no thread is started if the write delay is 0
// (if we only had a setter in the MVStore,
// the thread would need to be started in any case)
return set("writeDelay", writeDelay);
}
/** /**
* Open the store. * Open the store.
* *
......
...@@ -96,7 +96,7 @@ public class MVSecondaryIndex extends BaseIndex { ...@@ -96,7 +96,7 @@ public class MVSecondaryIndex extends BaseIndex {
} }
} }
try { try {
map.put(array, ValueLong.get(0)); map.put(array, ValueNull.INSTANCE);
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, table.getName()); throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, table.getName());
} }
......
...@@ -54,10 +54,11 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex { ...@@ -54,10 +54,11 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex {
private final String mapName; private final String mapName;
private TransactionMap<SpatialKey, Value> dataMap; private TransactionMap<SpatialKey, Value> dataMap;
private MVRTreeMap<VersionedValue> spatialMap; private MVRTreeMap<VersionedValue> spatialMap;
/** /**
* Constructor. * Constructor.
* *
* @param db the database
* @param table the table instance * @param table the table instance
* @param id the index id * @param id the index id
* @param indexName the index name * @param indexName the index name
...@@ -66,7 +67,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex { ...@@ -66,7 +67,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex {
*/ */
public MVSpatialIndex( public MVSpatialIndex(
Database db, MVTable table, int id, String indexName, Database db, MVTable table, int id, String indexName,
IndexColumn[] columns, IndexType indexType) { IndexColumn[] columns, IndexType indexType) {
if (columns.length != 1) { if (columns.length != 1) {
throw DbException.getUnsupportedException("Can only index one column"); throw DbException.getUnsupportedException("Can only index one column");
} }
...@@ -92,7 +93,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex { ...@@ -92,7 +93,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex {
mapName = "index." + getId(); mapName = "index." + getId();
ValueDataType vt = new ValueDataType(null, null, null); ValueDataType vt = new ValueDataType(null, null, null);
VersionedValueType valueType = new VersionedValueType(vt); VersionedValueType valueType = new VersionedValueType(vt);
MVRTreeMap.Builder<VersionedValue> mapBuilder = MVRTreeMap.Builder<VersionedValue> mapBuilder =
new MVRTreeMap.Builder<VersionedValue>(). new MVRTreeMap.Builder<VersionedValue>().
valueType(valueType); valueType(valueType);
spatialMap = db.getMvStore().getStore().openMap(mapName, mapBuilder); spatialMap = db.getMvStore().getStore().openMap(mapName, mapBuilder);
...@@ -144,7 +145,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex { ...@@ -144,7 +145,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex {
} }
} }
} }
private SpatialKey getKey(SearchRow r) { private SpatialKey getKey(SearchRow r) {
if (r == null) { if (r == null) {
return null; return null;
...@@ -171,7 +172,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex { ...@@ -171,7 +172,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex {
throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, table.getName()); throw DbException.get(ErrorCode.CONCURRENT_UPDATE_1, table.getName());
} }
} }
@Override @Override
public Cursor find(TableFilter filter, SearchRow first, SearchRow last) { public Cursor find(TableFilter filter, SearchRow first, SearchRow last) {
return find(filter.getSession()); return find(filter.getSession());
...@@ -200,7 +201,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex { ...@@ -200,7 +201,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex {
Iterator<SpatialKey> it = map.wrapIterator(cursor, false); Iterator<SpatialKey> it = map.wrapIterator(cursor, false);
return new MVStoreCursor(session, it); return new MVStoreCursor(session, it);
} }
private SpatialKey getEnvelope(SearchRow row) { private SpatialKey getEnvelope(SearchRow row) {
Value v = row.getValue(columnIds[0]); Value v = row.getValue(columnIds[0]);
Geometry g = ((ValueGeometry) v.convertTo(Value.GEOMETRY)).getGeometry(); Geometry g = ((ValueGeometry) v.convertTo(Value.GEOMETRY)).getGeometry();
...@@ -232,7 +233,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex { ...@@ -232,7 +233,7 @@ public class MVSpatialIndex extends BaseIndex implements SpatialIndex {
public double getCost(Session session, int[] masks, TableFilter filter, SortOrder sortOrder) { public double getCost(Session session, int[] masks, TableFilter filter, SortOrder sortOrder) {
return getCostRangeIndex(masks, table.getRowCountApproximation(), filter, sortOrder); return getCostRangeIndex(masks, table.getRowCountApproximation(), filter, sortOrder);
} }
@Override @Override
protected long getCostRangeIndex(int[] masks, long rowCount, TableFilter filter, SortOrder sortOrder) { protected long getCostRangeIndex(int[] masks, long rowCount, TableFilter filter, SortOrder sortOrder) {
rowCount += Constants.COST_ROW_OFFSET; rowCount += Constants.COST_ROW_OFFSET;
......
...@@ -399,7 +399,7 @@ public class MVTable extends TableBase { ...@@ -399,7 +399,7 @@ public class MVTable extends TableBase {
indexName, primaryIndex, indexType); indexName, primaryIndex, indexType);
} else if (indexType.isSpatial()) { } else if (indexType.isSpatial()) {
index = new MVSpatialIndex(session.getDatabase(), index = new MVSpatialIndex(session.getDatabase(),
this, indexId, this, indexId,
indexName, cols, indexType); indexName, cols, indexType);
} else { } else {
index = new MVSecondaryIndex(session.getDatabase(), index = new MVSecondaryIndex(session.getDatabase(),
......
...@@ -168,13 +168,13 @@ public class MVTableEngine implements TableEngine { ...@@ -168,13 +168,13 @@ public class MVTableEngine implements TableEngine {
/** /**
* Store all pending changes. * Store all pending changes.
*/ */
public void store() { public void flush() {
FileStore s = store.getFileStore(); FileStore s = store.getFileStore();
if (s == null || s.isReadOnly()) { if (s == null || s.isReadOnly()) {
return; return;
} }
if (!store.compact(50)) { if (!store.compact(50)) {
store.store(); store.commit();
} }
} }
...@@ -213,9 +213,7 @@ public class MVTableEngine implements TableEngine { ...@@ -213,9 +213,7 @@ public class MVTableEngine implements TableEngine {
Transaction t = session.getTransaction(); Transaction t = session.getTransaction();
t.setName(transactionName); t.setName(transactionName);
t.prepare(); t.prepare();
if (store.getFileStore() != null) { store.commit();
store.store();
}
} }
public ArrayList<InDoubtTransaction> getInDoubtTransactions() { public ArrayList<InDoubtTransaction> getInDoubtTransactions() {
...@@ -245,7 +243,7 @@ public class MVTableEngine implements TableEngine { ...@@ -245,7 +243,7 @@ public class MVTableEngine implements TableEngine {
* Force the changes to disk. * Force the changes to disk.
*/ */
public void sync() { public void sync() {
store(); flush();
store.sync(); store.sync();
} }
...@@ -316,9 +314,7 @@ public class MVTableEngine implements TableEngine { ...@@ -316,9 +314,7 @@ public class MVTableEngine implements TableEngine {
} else { } else {
transaction.rollback(); transaction.rollback();
} }
if (store.getFileStore() != null) { store.commit();
store.store();
}
this.state = state; this.state = state;
} }
......
...@@ -170,9 +170,7 @@ public class TransactionStore { ...@@ -170,9 +170,7 @@ public class TransactionStore {
public synchronized void close() { public synchronized void close() {
// to avoid losing transaction ids // to avoid losing transaction ids
settings.put(LAST_TRANSACTION_ID, "" + lastTransactionId); settings.put(LAST_TRANSACTION_ID, "" + lastTransactionId);
if (store.getFileStore() != null) { store.commit();
store.store();
}
} }
/** /**
...@@ -192,9 +190,7 @@ public class TransactionStore { ...@@ -192,9 +190,7 @@ public class TransactionStore {
private void commitIfNeeded() { private void commitIfNeeded() {
if (store.getUnsavedPageCount() > MAX_UNSAVED_PAGES) { if (store.getUnsavedPageCount() > MAX_UNSAVED_PAGES) {
if (store.getFileStore() != null) { store.commit();
store.store();
}
} }
} }
...@@ -354,10 +350,8 @@ public class TransactionStore { ...@@ -354,10 +350,8 @@ public class TransactionStore {
if (t.getId() == firstOpenTransaction) { if (t.getId() == firstOpenTransaction) {
firstOpenTransaction = -1; firstOpenTransaction = -1;
} }
if (store.getWriteDelay() == 0) { if (store.getAutoCommitDelay() == 0) {
if (store.getFileStore() != null) { store.commit();
store.store();
}
return; return;
} }
// to avoid having to store the transaction log, // to avoid having to store the transaction log,
...@@ -365,10 +359,10 @@ public class TransactionStore { ...@@ -365,10 +359,10 @@ public class TransactionStore {
// and if there have been many changes, store them now // and if there have been many changes, store them now
if (undoLog.isEmpty()) { if (undoLog.isEmpty()) {
int unsaved = store.getUnsavedPageCount(); int unsaved = store.getUnsavedPageCount();
int max = store.getUnsavedPageCountMax(); int max = store.getAutoCommitPageCount();
// save at 3/4 capacity // save at 3/4 capacity
if (unsaved * 4 > max * 3) { if (unsaved * 4 > max * 3) {
store.store(); store.commit();
} }
} }
} }
...@@ -658,7 +652,15 @@ public class TransactionStore { ...@@ -658,7 +652,15 @@ public class TransactionStore {
int mapId = map.getId(); int mapId = map.getId();
return new TransactionMap<K, V>(this, map, mapId); return new TransactionMap<K, V>(this, map, mapId);
} }
/**
* Open the transactional version of the given map.
*
* @param <K> the key type
* @param <V> the value type
* @param map the base map
* @return the transactional map
*/
public <K, V> TransactionMap<K, V> openMap(MVMap<K, VersionedValue> map) { public <K, V> TransactionMap<K, V> openMap(MVMap<K, VersionedValue> map) {
checkNotClosed(); checkNotClosed();
int mapId = map.getId(); int mapId = map.getId();
...@@ -726,9 +728,11 @@ public class TransactionStore { ...@@ -726,9 +728,11 @@ public class TransactionStore {
DataUtils.ERROR_CLOSED, "Transaction is closed"); DataUtils.ERROR_CLOSED, "Transaction is closed");
} }
} }
/** /**
* Remove the map. * Remove the map.
*
* @param map the map
*/ */
public <K, V> void removeMap(TransactionMap<K, V> map) { public <K, V> void removeMap(TransactionMap<K, V> map) {
store.store.removeMap(map.map); store.store.removeMap(map.map);
...@@ -1258,7 +1262,7 @@ public class TransactionStore { ...@@ -1258,7 +1262,7 @@ public class TransactionStore {
} }
}; };
} }
public Transaction getTransaction() { public Transaction getTransaction() {
return transaction; return transaction;
} }
......
...@@ -56,33 +56,32 @@ public class TestConcurrent extends TestMVStore { ...@@ -56,33 +56,32 @@ public class TestConcurrent extends TestMVStore {
testConcurrentWrite(); testConcurrentWrite();
testConcurrentRead(); testConcurrentRead();
} }
private void testConcurrentFree() throws InterruptedException { private void testConcurrentFree() throws InterruptedException {
String fileName = "memFS:testConcurrentFree.h3"; String fileName = "memFS:testConcurrentFree.h3";
for (int test = 0; test < 10; test++) { for (int test = 0; test < 10; test++) {
FileUtils.delete(fileName); FileUtils.delete(fileName);
final MVStore s1 = new MVStore.Builder(). final MVStore s1 = new MVStore.Builder().
fileName(fileName).writeDelay(-1).open(); fileName(fileName).autoCommitDisabled().open();
s1.setRetentionTime(0); s1.setRetentionTime(0);
final int count = 200; final int count = 200;
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
MVMap<Integer, Integer> m = s1.openMap("d" + i); MVMap<Integer, Integer> m = s1.openMap("d" + i);
m.put(1, 1); m.put(1, 1);
if (i % 2 == 0) { if (i % 2 == 0) {
s1.store(); s1.commit();
} }
} }
s1.store();
s1.close(); s1.close();
final MVStore s = new MVStore.Builder(). final MVStore s = new MVStore.Builder().
fileName(fileName).writeDelay(-1).open(); fileName(fileName).autoCommitDisabled().open();
s.setRetentionTime(0); s.setRetentionTime(0);
final ArrayList<MVMap<Integer, Integer>> list = New.arrayList(); final ArrayList<MVMap<Integer, Integer>> list = New.arrayList();
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
MVMap<Integer, Integer> m = s.openMap("d" + i); MVMap<Integer, Integer> m = s.openMap("d" + i);
list.add(m); list.add(m);
} }
final AtomicInteger counter = new AtomicInteger(); final AtomicInteger counter = new AtomicInteger();
Task task = new Task() { Task task = new Task() {
@Override @Override
...@@ -113,8 +112,8 @@ public class TestConcurrent extends TestMVStore { ...@@ -113,8 +112,8 @@ public class TestConcurrent extends TestMVStore {
} }
} }
task.get(); task.get();
s.store(); s.commit();
MVMap<String, String> meta = s.getMetaMap(); MVMap<String, String> meta = s.getMetaMap();
int chunkCount = 0; int chunkCount = 0;
for (String k : meta.keyList()) { for (String k : meta.keyList()) {
...@@ -142,7 +141,7 @@ public class TestConcurrent extends TestMVStore { ...@@ -142,7 +141,7 @@ public class TestConcurrent extends TestMVStore {
public void call() throws Exception { public void call() throws Exception {
while (!stop) { while (!stop) {
counter.incrementAndGet(); counter.incrementAndGet();
s.store(); s.commit();
} }
} }
}; };
...@@ -172,7 +171,7 @@ public class TestConcurrent extends TestMVStore { ...@@ -172,7 +171,7 @@ public class TestConcurrent extends TestMVStore {
public void call() throws Exception { public void call() throws Exception {
while (!stop) { while (!stop) {
s.setStoreVersion(counter.incrementAndGet()); s.setStoreVersion(counter.incrementAndGet());
s.store(); s.commit();
} }
} }
}; };
...@@ -270,9 +269,9 @@ public class TestConcurrent extends TestMVStore { ...@@ -270,9 +269,9 @@ public class TestConcurrent extends TestMVStore {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
map.put(i, new byte[100 * r.nextInt(100)]); map.put(i, new byte[100 * r.nextInt(100)]);
} }
s.store(); s.commit();
map.clear(); map.clear();
s.store(); s.commit();
long len = s.getFileStore().size(); long len = s.getFileStore().size();
if (len > 1024 * 1024) { if (len > 1024 * 1024) {
// slow down writing a lot // slow down writing a lot
...@@ -322,6 +321,7 @@ public class TestConcurrent extends TestMVStore { ...@@ -322,6 +321,7 @@ public class TestConcurrent extends TestMVStore {
private void testConcurrentIterate() { private void testConcurrentIterate() {
MVStore s = new MVStore.Builder().pageSplitSize(3).open(); MVStore s = new MVStore.Builder().pageSplitSize(3).open();
s.setVersionsToKeep(100);
final MVMap<Integer, Integer> map = s.openMap("test"); final MVMap<Integer, Integer> map = s.openMap("test");
final int len = 10; final int len = 10;
final Random r = new Random(); final Random r = new Random();
...@@ -343,7 +343,6 @@ public class TestConcurrent extends TestMVStore { ...@@ -343,7 +343,6 @@ public class TestConcurrent extends TestMVStore {
Iterator<Integer> it = map.keyIterator(r.nextInt(len)); Iterator<Integer> it = map.keyIterator(r.nextInt(len));
long old = s.getCurrentVersion(); long old = s.getCurrentVersion();
s.commit(); s.commit();
s.setRetainVersion(old - 100);
while (map.getVersion() == old) { while (map.getVersion() == old) {
Thread.yield(); Thread.yield();
} }
......
...@@ -84,9 +84,8 @@ public class TestKillProcessWhileWriting extends TestBase { ...@@ -84,9 +84,8 @@ public class TestKillProcessWhileWriting extends TestBase {
s = new MVStore.Builder(). s = new MVStore.Builder().
fileName(fileName). fileName(fileName).
pageSplitSize(50). pageSplitSize(50).
writeDelay(0). autoCommitDisabled().
open(); open();
s.setWriteDelay(0);
m = s.openMap("data"); m = s.openMap("data");
Random r = new Random(seed); Random r = new Random(seed);
int op = 0; int op = 0;
...@@ -107,7 +106,7 @@ public class TestKillProcessWhileWriting extends TestBase { ...@@ -107,7 +106,7 @@ public class TestKillProcessWhileWriting extends TestBase {
m.remove(k); m.remove(k);
break; break;
case 6: case 6:
s.store(); s.commit();
break; break;
case 7: case 7:
s.compact(80); s.compact(80);
...@@ -120,12 +119,12 @@ public class TestKillProcessWhileWriting extends TestBase { ...@@ -120,12 +119,12 @@ public class TestKillProcessWhileWriting extends TestBase {
s = new MVStore.Builder(). s = new MVStore.Builder().
fileName(fileName). fileName(fileName).
pageSplitSize(50). pageSplitSize(50).
writeDelay(0).open(); autoCommitDisabled().
open();
m = s.openMap("data"); m = s.openMap("data");
break; break;
} }
} }
s.store();
s.close(); s.close();
return 0; return 0;
} catch (Exception e) { } catch (Exception e) {
......
...@@ -54,7 +54,7 @@ public class TestMVRTree extends TestMVStore { ...@@ -54,7 +54,7 @@ public class TestMVRTree extends TestMVStore {
testRandom(); testRandom();
testRandomFind(); testRandomFind();
} }
private void testSpatialKey() { private void testSpatialKey() {
SpatialKey a0 = new SpatialKey(0, 1, 2, 3, 4); SpatialKey a0 = new SpatialKey(0, 1, 2, 3, 4);
SpatialKey a1 = new SpatialKey(0, 1, 2, 3, 4); SpatialKey a1 = new SpatialKey(0, 1, 2, 3, 4);
...@@ -119,7 +119,7 @@ public class TestMVRTree extends TestMVStore { ...@@ -119,7 +119,7 @@ public class TestMVRTree extends TestMVStore {
SpatialKey k = new SpatialKey(i, x - p, x + p, y - p, y + p); SpatialKey k = new SpatialKey(i, x - p, x + p, y - p, y + p);
r.add(k, "" + i); r.add(k, "" + i);
if (i > 0 && (i % len / 10) == 0) { if (i > 0 && (i % len / 10) == 0) {
s.store(); s.commit();
} }
if (i > 0 && (i % 10000) == 0) { if (i > 0 && (i % 10000) == 0) {
render(r, getBaseDir() + "/test.png"); render(r, getBaseDir() + "/test.png");
...@@ -127,7 +127,6 @@ public class TestMVRTree extends TestMVStore { ...@@ -127,7 +127,6 @@ public class TestMVRTree extends TestMVStore {
} }
// System.out.println(prof.getTop(5)); // System.out.println(prof.getTop(5));
// System.out.println("add: " + (System.currentTimeMillis() - t)); // System.out.println("add: " + (System.currentTimeMillis() - t));
s.store();
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
r = s.openMap("data", r = s.openMap("data",
......
...@@ -48,6 +48,8 @@ public class TestMVStore extends TestBase { ...@@ -48,6 +48,8 @@ public class TestMVStore extends TestBase {
public void test() throws Exception { public void test() throws Exception {
FileUtils.deleteRecursive(getBaseDir(), true); FileUtils.deleteRecursive(getBaseDir(), true);
FileUtils.createDirectories(getBaseDir()); FileUtils.createDirectories(getBaseDir());
testRollback();
testVersionsToKeep();
testRemoveMap(); testRemoveMap();
testIsEmpty(); testIsEmpty();
testOffHeapStorage(); testOffHeapStorage();
...@@ -97,6 +99,26 @@ public class TestMVStore extends TestBase { ...@@ -97,6 +99,26 @@ public class TestMVStore extends TestBase {
testLargerThan2G(); testLargerThan2G();
} }
private void testVersionsToKeep() throws Exception {
MVStore s = new MVStore.Builder().open();
MVMap<Integer, Integer> map;
map = s.openMap("data");
for (int i = 0; i < 20; i++) {
long version = s.getCurrentVersion();
map.put(i, i);
s.commit();
if (version >= 6) {
map.openVersion(version - 6);
try {
map.openVersion(version - 7);
fail();
} catch (IllegalArgumentException e) {
// expected
}
}
}
}
private void testRemoveMap() throws Exception { private void testRemoveMap() throws Exception {
String fileName = getBaseDir() + "/testCloseMap.h3"; String fileName = getBaseDir() + "/testCloseMap.h3";
FileUtils.delete(fileName); FileUtils.delete(fileName);
...@@ -108,15 +130,14 @@ public class TestMVStore extends TestBase { ...@@ -108,15 +130,14 @@ public class TestMVStore extends TestBase {
map = s.openMap("data"); map = s.openMap("data");
map.put(1, 1); map.put(1, 1);
assertEquals(1, map.get(1).intValue()); assertEquals(1, map.get(1).intValue());
s.store(); s.commit();
s.removeMap(map); s.removeMap(map);
s.store(); s.commit();
map = s.openMap("data"); map = s.openMap("data");
assertTrue(map.isEmpty()); assertTrue(map.isEmpty());
map.put(2, 2); map.put(2, 2);
s.store();
s.close(); s.close();
} }
...@@ -145,7 +166,7 @@ public class TestMVStore extends TestBase { ...@@ -145,7 +166,7 @@ public class TestMVStore extends TestBase {
Map<Integer, String> map = s.openMap("data"); Map<Integer, String> map = s.openMap("data");
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
map.put(i, "Hello " + i); map.put(i, "Hello " + i);
s.store(); s.commit();
} }
assertTrue(1000 < offHeap.getWriteCount()); assertTrue(1000 < offHeap.getWriteCount());
s.close(); s.close();
...@@ -173,7 +194,6 @@ public class TestMVStore extends TestBase { ...@@ -173,7 +194,6 @@ public class TestMVStore extends TestBase {
header.put("format", "2"); header.put("format", "2");
MVMap<Integer, String> m = s.openMap("data"); MVMap<Integer, String> m = s.openMap("data");
m.put(0, "Hello World"); m.put(0, "Hello World");
s.store();
s.close(); s.close();
try { try {
...@@ -218,12 +238,12 @@ public class TestMVStore extends TestBase { ...@@ -218,12 +238,12 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
m = s.openMap("data" + i); m = s.openMap("data" + i);
m.put(0, "Hello World"); m.put(0, "Hello World");
s.store(); s.commit();
} }
for (int i = 0; i < 10; i += 2) { for (int i = 0; i < 10; i += 2) {
m = s.openMap("data" + i); m = s.openMap("data" + i);
s.removeMap(m); s.removeMap(m);
s.store(); s.commit();
} }
long sizeOld = s.getFileStore().size(); long sizeOld = s.getFileStore().size();
s.compactMoveChunks(); s.compactMoveChunks();
...@@ -248,7 +268,7 @@ public class TestMVStore extends TestBase { ...@@ -248,7 +268,7 @@ public class TestMVStore extends TestBase {
}). }).
open(); open();
s.setWriteDelay(2); s.setAutoCommitDelay(2);
MVMap<Integer, String> m; MVMap<Integer, String> m;
m = s.openMap("data"); m = s.openMap("data");
s.getFileStore().getFile().close(); s.getFileStore().getFile().close();
...@@ -313,7 +333,7 @@ public class TestMVStore extends TestBase { ...@@ -313,7 +333,7 @@ public class TestMVStore extends TestBase {
for (int bs = 0; bs <= 1; bs++) { for (int bs = 0; bs <= 1; bs++) {
s = new MVStore.Builder(). s = new MVStore.Builder().
fileName(fileName). fileName(fileName).
writeBufferSize(bs). autoCommitBufferSize(bs).
open(); open();
m = s.openMap("data"); m = s.openMap("data");
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
...@@ -332,7 +352,7 @@ public class TestMVStore extends TestBase { ...@@ -332,7 +352,7 @@ public class TestMVStore extends TestBase {
assertTrue(m.containsKey(1)); assertTrue(m.containsKey(1));
m.put(-1, data); m.put(-1, data);
s.store(); s.commit();
m.put(-2, data); m.put(-2, data);
s.close(); s.close();
...@@ -354,15 +374,15 @@ public class TestMVStore extends TestBase { ...@@ -354,15 +374,15 @@ public class TestMVStore extends TestBase {
FileUtils.delete(fileName); FileUtils.delete(fileName);
s = new MVStore.Builder(). s = new MVStore.Builder().
autoCommitDisabled().
fileName(fileName).open(); fileName(fileName).open();
s.setWriteDelay(0);
m = s.openMap("data"); m = s.openMap("data");
m.put(1, "1"); m.put(1, "1");
s.commit(); s.commit();
s.close(); s.close();
s = new MVStore.Builder(). s = new MVStore.Builder().
autoCommitDisabled().
fileName(fileName).open(); fileName(fileName).open();
s.setWriteDelay(0);
m = s.openMap("data"); m = s.openMap("data");
assertEquals(1, m.size()); assertEquals(1, m.size());
s.close(); s.close();
...@@ -371,10 +391,10 @@ public class TestMVStore extends TestBase { ...@@ -371,10 +391,10 @@ public class TestMVStore extends TestBase {
s = new MVStore.Builder(). s = new MVStore.Builder().
fileName(fileName). fileName(fileName).
open(); open();
s.setWriteDelay(1); s.setAutoCommitDelay(1);
m = s.openMap("data"); m = s.openMap("data");
m.put(1, "Hello"); m.put(1, "Hello");
s.store(); s.commit();
long v = s.getCurrentVersion(); long v = s.getCurrentVersion();
m.put(2, "World."); m.put(2, "World.");
Thread.sleep(5); Thread.sleep(5);
...@@ -383,12 +403,11 @@ public class TestMVStore extends TestBase { ...@@ -383,12 +403,11 @@ public class TestMVStore extends TestBase {
s = new MVStore.Builder(). s = new MVStore.Builder().
fileName(fileName). fileName(fileName).
open(); open();
s.setWriteDelay(1); s.setAutoCommitDelay(1);
m = s.openMap("data"); m = s.openMap("data");
assertEquals("World.", m.get(2)); assertEquals("World.", m.get(2));
m.put(2, "World"); m.put(2, "World");
s.commit(); s.commit();
s.store();
v = s.getCurrentVersion(); v = s.getCurrentVersion();
m.put(3, "!"); m.put(3, "!");
...@@ -433,7 +452,6 @@ public class TestMVStore extends TestBase { ...@@ -433,7 +452,6 @@ public class TestMVStore extends TestBase {
m = s.openMap("test"); m = s.openMap("test");
m.put(1, "Hello"); m.put(1, "Hello");
assertEquals("Hello", m.get(1)); assertEquals("Hello", m.get(1));
s.store();
s.close(); s.close();
passwordChars = "008".toCharArray(); passwordChars = "008".toCharArray();
...@@ -485,7 +503,6 @@ public class TestMVStore extends TestBase { ...@@ -485,7 +503,6 @@ public class TestMVStore extends TestBase {
int format = Integer.parseInt(header.get("format")); int format = Integer.parseInt(header.get("format"));
assertEquals(1, format); assertEquals(1, format);
header.put("format", Integer.toString(format + 1)); header.put("format", Integer.toString(format + 1));
s.store();
s.close(); s.close();
try { try {
openStore(fileName).close(); openStore(fileName).close();
...@@ -503,9 +520,8 @@ public class TestMVStore extends TestBase { ...@@ -503,9 +520,8 @@ public class TestMVStore extends TestBase {
MVStore s = openStore(fileName); MVStore s = openStore(fileName);
MVMap<Integer, Integer> m = s.openMap("test"); MVMap<Integer, Integer> m = s.openMap("test");
m.put(1, 1); m.put(1, 1);
s.store(); s.commit();
s.removeMap(m); s.removeMap(m);
s.store();
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
m = s.openMap("test"); m = s.openMap("test");
...@@ -548,17 +564,16 @@ public class TestMVStore extends TestBase { ...@@ -548,17 +564,16 @@ public class TestMVStore extends TestBase {
MVMap<Integer, String> map; MVMap<Integer, String> map;
s = new MVStore.Builder(). s = new MVStore.Builder().
fileName(fileName). fileName(fileName).
writeDelay(-1). autoCommitDisabled().
compressData().open(); compressData().open();
map = s.openMap("test"); map = s.openMap("test");
// add 10 MB of data // add 10 MB of data
for (int i = 0; i < 1024; i++) { for (int i = 0; i < 1024; i++) {
map.put(i, new String(new char[10240])); map.put(i, new String(new char[10240]));
} }
s.store();
s.close(); s.close();
int[] expectedReadsForCacheSize = { int[] expectedReadsForCacheSize = {
3405, 2590, 1924, 1440, 1108, 956, 918 3413, 2590, 1924, 1440, 1112, 956, 918
}; };
for (int cacheSize = 0; cacheSize <= 6; cacheSize += 4) { for (int cacheSize = 0; cacheSize <= 6; cacheSize += 4) {
s = new MVStore.Builder(). s = new MVStore.Builder().
...@@ -613,7 +628,6 @@ public class TestMVStore extends TestBase { ...@@ -613,7 +628,6 @@ public class TestMVStore extends TestBase {
s.getStoreHeader().put("test", "123"); s.getStoreHeader().put("test", "123");
MVMap<Integer, Integer> map = s.openMap("test"); MVMap<Integer, Integer> map = s.openMap("test");
map.put(10, 100); map.put(10, 100);
s.store();
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
assertEquals("123", s.getStoreHeader().get("test")); assertEquals("123", s.getStoreHeader().get("test"));
...@@ -630,14 +644,13 @@ public class TestMVStore extends TestBase { ...@@ -630,14 +644,13 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
map = s.openMap("test" + i); map = s.openMap("test" + i);
map.put(0, new byte[1000]); map.put(0, new byte[1000]);
s.store(); s.commit();
} }
FileStore fs = s.getFileStore(); FileStore fs = s.getFileStore();
long size = fs.getFile().size(); long size = fs.getFile().size();
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
map = s.openMap("test" + i); map = s.openMap("test" + i);
s.removeMap(map); s.removeMap(map);
s.store();
s.commit(); s.commit();
s.compact(100); s.compact(100);
if (fs.getFile().size() <= size) { if (fs.getFile().size() <= size) {
...@@ -818,14 +831,13 @@ public class TestMVStore extends TestBase { ...@@ -818,14 +831,13 @@ public class TestMVStore extends TestBase {
assertEquals(0, s.getCurrentVersion()); assertEquals(0, s.getCurrentVersion());
assertEquals(0, s.getStoreVersion()); assertEquals(0, s.getStoreVersion());
s.setStoreVersion(0); s.setStoreVersion(0);
s.store(); s.commit();
s.setStoreVersion(1); s.setStoreVersion(1);
s.closeImmediately(); s.closeImmediately();
s = MVStore.open(fileName); s = MVStore.open(fileName);
assertEquals(1, s.getCurrentVersion()); assertEquals(1, s.getCurrentVersion());
assertEquals(0, s.getStoreVersion()); assertEquals(0, s.getStoreVersion());
s.setStoreVersion(1); s.setStoreVersion(1);
s.store();
s.close(); s.close();
s = MVStore.open(fileName); s = MVStore.open(fileName);
assertEquals(2, s.getCurrentVersion()); assertEquals(2, s.getCurrentVersion());
...@@ -866,7 +878,6 @@ public class TestMVStore extends TestBase { ...@@ -866,7 +878,6 @@ public class TestMVStore extends TestBase {
map.put(1, "Hello"); map.put(1, "Hello");
map.put("2", 200); map.put("2", 200);
map.put(new Object[1], new Object[]{1, "2"}); map.put(new Object[1], new Object[]{1, "2"});
s.store();
s.close(); s.close();
s = new MVStore.Builder().fileName(fileName).open(); s = new MVStore.Builder().fileName(fileName).open();
...@@ -958,7 +969,7 @@ public class TestMVStore extends TestBase { ...@@ -958,7 +969,7 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
Integer x = m.get("value"); Integer x = m.get("value");
m.put("value", x == null ? 0 : x + 1); m.put("value", x == null ? 0 : x + 1);
s.store(); s.commit();
} }
s.close(); s.close();
} }
...@@ -972,7 +983,7 @@ public class TestMVStore extends TestBase { ...@@ -972,7 +983,7 @@ public class TestMVStore extends TestBase {
for (int op = 0; op <= 1; op++) { for (int op = 0; op <= 1; op++) {
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
s = openStore(null); s = openStore(null);
s.setRetainVersion(0); s.setVersionsToKeep(Integer.MAX_VALUE);
MVMap<String, String> m; MVMap<String, String> m;
m = s.openMap("data"); m = s.openMap("data");
for (int j = 0; j < 5; j++) { for (int j = 0; j < 5; j++) {
...@@ -1021,7 +1032,7 @@ public class TestMVStore extends TestBase { ...@@ -1021,7 +1032,7 @@ public class TestMVStore extends TestBase {
assertEquals("World", mOld.get("2")); assertEquals("World", mOld.get("2"));
assertTrue(mOld.isReadOnly()); assertTrue(mOld.isReadOnly());
s.getCurrentVersion(); s.getCurrentVersion();
long old3 = s.store(); long old3 = s.commit();
// the old version is still available // the old version is still available
assertEquals("Hello", mOld.get("1")); assertEquals("Hello", mOld.get("1"));
...@@ -1033,7 +1044,6 @@ public class TestMVStore extends TestBase { ...@@ -1033,7 +1044,6 @@ public class TestMVStore extends TestBase {
m.put("1", "Hi"); m.put("1", "Hi");
assertEquals("Welt", m.remove("2")); assertEquals("Welt", m.remove("2"));
s.store();
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
...@@ -1064,14 +1074,12 @@ public class TestMVStore extends TestBase { ...@@ -1064,14 +1074,12 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
m.put(i, "Hello World"); m.put(i, "Hello World");
} }
s.store();
s.close(); s.close();
long len = FileUtils.size(fileName); long len = FileUtils.size(fileName);
s = openStore(fileName); s = openStore(fileName);
s.setRetentionTime(0); s.setRetentionTime(0);
m = s.openMap("data"); m = s.openMap("data");
m.clear(); m.clear();
s.store();
s.compact(100); s.compact(100);
s.close(); s.close();
long len2 = FileUtils.size(fileName); long len2 = FileUtils.size(fileName);
...@@ -1091,7 +1099,7 @@ public class TestMVStore extends TestBase { ...@@ -1091,7 +1099,7 @@ public class TestMVStore extends TestBase {
} }
assertEquals(1000, m.size()); assertEquals(1000, m.size());
assertEquals(285, s.getUnsavedPageCount()); assertEquals(285, s.getUnsavedPageCount());
s.store(); s.commit();
assertEquals(2, s.getFileStore().getWriteCount()); assertEquals(2, s.getFileStore().getWriteCount());
s.close(); s.close();
...@@ -1099,13 +1107,27 @@ public class TestMVStore extends TestBase { ...@@ -1099,13 +1107,27 @@ public class TestMVStore extends TestBase {
m = s.openMap("data"); m = s.openMap("data");
m.clear(); m.clear();
assertEquals(0, m.size()); assertEquals(0, m.size());
s.store(); s.commit();
// ensure only nodes are read, but not leaves // ensure only nodes are read, but not leaves
assertEquals(42, s.getFileStore().getReadCount()); assertEquals(42, s.getFileStore().getReadCount());
assertEquals(1, s.getFileStore().getWriteCount()); assertEquals(1, s.getFileStore().getWriteCount());
s.close(); s.close();
} }
private void testRollback() {
MVStore s = MVStore.open(null);
MVMap<Integer, Integer> m = s.openMap("m");
m.put(1, -1);
s.commit();
for (int i = 0; i < 10; i++) {
m.put(1, i);
s.rollback();
assertEquals(i - 1, m.get(1).intValue());
m.put(1, i);
s.commit();
}
}
private void testRollbackStored() { private void testRollbackStored() {
String fileName = getBaseDir() + "/testRollback.h3"; String fileName = getBaseDir() + "/testRollback.h3";
FileUtils.delete(fileName); FileUtils.delete(fileName);
...@@ -1129,7 +1151,7 @@ public class TestMVStore extends TestBase { ...@@ -1129,7 +1151,7 @@ public class TestMVStore extends TestBase {
// so a new version is created // so a new version is created
m.put("1", "Hello"); m.put("1", "Hello");
long v2 = s.store(); long v2 = s.commit();
assertEquals(2, v2); assertEquals(2, v2);
assertEquals(2, s.getCurrentVersion()); assertEquals(2, s.getCurrentVersion());
assertFalse(s.hasUnsavedChanges()); assertFalse(s.hasUnsavedChanges());
...@@ -1155,7 +1177,7 @@ public class TestMVStore extends TestBase { ...@@ -1155,7 +1177,7 @@ public class TestMVStore extends TestBase {
assertNull(meta.get("name.data1")); assertNull(meta.get("name.data1"));
assertNull(m0.get("1")); assertNull(m0.get("1"));
assertEquals("Hello", m.get("1")); assertEquals("Hello", m.get("1"));
assertEquals(2, s.store()); assertEquals(2, s.commit());
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
...@@ -1179,7 +1201,6 @@ public class TestMVStore extends TestBase { ...@@ -1179,7 +1201,6 @@ public class TestMVStore extends TestBase {
assertEquals(3, s.getCurrentVersion()); assertEquals(3, s.getCurrentVersion());
m = s.openMap("data"); m = s.openMap("data");
m.put("1", "Hi"); m.put("1", "Hi");
s.store();
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
...@@ -1243,7 +1264,7 @@ public class TestMVStore extends TestBase { ...@@ -1243,7 +1264,7 @@ public class TestMVStore extends TestBase {
MVMap<String, String> data = s.openMap("data"); MVMap<String, String> data = s.openMap("data");
data.put("1", "Hello"); data.put("1", "Hello");
data.put("2", "World"); data.put("2", "World");
s.store(); s.commit();
assertEquals(1, s.getCurrentVersion()); assertEquals(1, s.getCurrentVersion());
assertTrue(m.containsKey("chunk.1")); assertTrue(m.containsKey("chunk.1"));
assertFalse(m.containsKey("chunk.2")); assertFalse(m.containsKey("chunk.2"));
...@@ -1252,7 +1273,7 @@ public class TestMVStore extends TestBase { ...@@ -1252,7 +1273,7 @@ public class TestMVStore extends TestBase {
assertEquals("name:data", m.get("map." + id)); assertEquals("name:data", m.get("map." + id));
assertTrue(m.containsKey("chunk.1")); assertTrue(m.containsKey("chunk.1"));
assertEquals("Hello", data.put("1", "Hallo")); assertEquals("Hello", data.put("1", "Hallo"));
s.store(); s.commit();
assertEquals("name:data", m.get("map." + id)); assertEquals("name:data", m.get("map." + id));
assertTrue(m.get("root.1").length() > 0); assertTrue(m.get("root.1").length() > 0);
assertTrue(m.containsKey("chunk.1")); assertTrue(m.containsKey("chunk.1"));
...@@ -1324,10 +1345,9 @@ public class TestMVStore extends TestBase { ...@@ -1324,10 +1345,9 @@ public class TestMVStore extends TestBase {
m.put(i, o); m.put(i, o);
i++; i++;
if (i % 10000 == 0) { if (i % 10000 == 0) {
s.store(); s.commit();
} }
} }
s.store();
s.close(); s.close();
// System.out.println(prof.getTop(5)); // System.out.println(prof.getTop(5));
// System.out.println("store time " + (System.currentTimeMillis() - t)); // System.out.println("store time " + (System.currentTimeMillis() - t));
...@@ -1356,7 +1376,7 @@ public class TestMVStore extends TestBase { ...@@ -1356,7 +1376,7 @@ public class TestMVStore extends TestBase {
// p = new Profiler(); // p = new Profiler();
//p.startCollecting(); //p.startCollecting();
// t = System.currentTimeMillis(); // t = System.currentTimeMillis();
s.store(); s.commit();
// System.out.println("store: " + (System.currentTimeMillis() - t)); // System.out.println("store: " + (System.currentTimeMillis() - t));
// System.out.println(p.getTop(5)); // System.out.println(p.getTop(5));
assertEquals("hello 0", m.remove(0)); assertEquals("hello 0", m.remove(0));
...@@ -1364,7 +1384,6 @@ public class TestMVStore extends TestBase { ...@@ -1364,7 +1384,6 @@ public class TestMVStore extends TestBase {
for (int i = 1; i < count; i++) { for (int i = 1; i < count; i++) {
assertEquals("hello " + i, m.get(i)); assertEquals("hello " + i, m.get(i));
} }
s.store();
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
...@@ -1376,7 +1395,7 @@ public class TestMVStore extends TestBase { ...@@ -1376,7 +1395,7 @@ public class TestMVStore extends TestBase {
for (int i = 1; i < count; i++) { for (int i = 1; i < count; i++) {
m.remove(i); m.remove(i);
} }
s.store(); s.commit();
assertNull(m.get(0)); assertNull(m.get(0));
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
assertNull(m.get(i)); assertNull(m.get(i));
...@@ -1394,7 +1413,7 @@ public class TestMVStore extends TestBase { ...@@ -1394,7 +1413,7 @@ public class TestMVStore extends TestBase {
for (int i = j * factor; i < 10 * factor; i++) { for (int i = j * factor; i < 10 * factor; i++) {
m.put(i, "Hello" + j); m.put(i, "Hello" + j);
} }
s.store(); s.commit();
} }
s.close(); s.close();
...@@ -1450,7 +1469,6 @@ public class TestMVStore extends TestBase { ...@@ -1450,7 +1469,6 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
m.put(j + i, "Hello " + j); m.put(j + i, "Hello " + j);
} }
s.store();
s.compact(80); s.compact(80);
s.close(); s.close();
long len = FileUtils.size(fileName); long len = FileUtils.size(fileName);
...@@ -1468,7 +1486,6 @@ public class TestMVStore extends TestBase { ...@@ -1468,7 +1486,6 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
m.remove(i); m.remove(i);
} }
s.store();
s.compact(80); s.compact(80);
s.close(); s.close();
// len = FileUtils.size(fileName); // len = FileUtils.size(fileName);
...@@ -1492,12 +1509,11 @@ public class TestMVStore extends TestBase { ...@@ -1492,12 +1509,11 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
m.put(i, "Hello"); m.put(i, "Hello");
} }
s.store(); s.commit();
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
assertEquals("Hello", m.get(i)); assertEquals("Hello", m.get(i));
assertEquals("Hello", m.remove(i)); assertEquals("Hello", m.remove(i));
} }
s.store();
s.close(); s.close();
long len = FileUtils.size(fileName); long len = FileUtils.size(fileName);
if (initialLength == 0) { if (initialLength == 0) {
...@@ -1584,7 +1600,6 @@ public class TestMVStore extends TestBase { ...@@ -1584,7 +1600,6 @@ public class TestMVStore extends TestBase {
si.put("Test", 10); si.put("Test", 10);
MVMap<String, String> ss = s.openMap("stringString"); MVMap<String, String> ss = s.openMap("stringString");
ss.put("Hello", "World"); ss.put("Hello", "World");
s.store();
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
is = s.openMap("intString"); is = s.openMap("intString");
...@@ -1608,7 +1623,7 @@ public class TestMVStore extends TestBase { ...@@ -1608,7 +1623,7 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
m.put(i, "hello " + i); m.put(i, "hello " + i);
} }
s.store(); s.commit();
it = m.keyIterator(null); it = m.keyIterator(null);
it.next(); it.next();
assertThrows(UnsupportedOperationException.class, it).remove(); assertThrows(UnsupportedOperationException.class, it).remove();
...@@ -1639,7 +1654,6 @@ public class TestMVStore extends TestBase { ...@@ -1639,7 +1654,6 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
m.put(i, "hello " + i); m.put(i, "hello " + i);
} }
s.store();
// closing twice should be fine // closing twice should be fine
s.close(); s.close();
s.close(); s.close();
...@@ -1653,14 +1667,13 @@ public class TestMVStore extends TestBase { ...@@ -1653,14 +1667,13 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
m.put(i, "hello " + i); m.put(i, "hello " + i);
} }
s.store(); s.commit();
assertEquals("hello 0", m.remove(0)); assertEquals("hello 0", m.remove(0));
assertNull(m.get(0)); assertNull(m.get(0));
for (int i = 1; i < 3; i++) { for (int i = 1; i < 3; i++) {
assertEquals("hello " + i, m.get(i)); assertEquals("hello " + i, m.get(i));
} }
s.store();
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
...@@ -1686,8 +1699,7 @@ public class TestMVStore extends TestBase { ...@@ -1686,8 +1699,7 @@ public class TestMVStore extends TestBase {
for (int i = 0;; i++) { for (int i = 0;; i++) {
map.put(i, data); map.put(i, data);
if (i % 10000 == 0) { if (i % 10000 == 0) {
long version = store.commit(); store.commit();
store.setRetainVersion(version);
long time = System.currentTimeMillis(); long time = System.currentTimeMillis();
if (time - last > 2000) { if (time - last > 2000) {
long mb = store.getFileStore().size() / 1024 / 1024; long mb = store.getFileStore().size() / 1024 / 1024;
......
...@@ -92,7 +92,7 @@ public class TestRandomMapOps extends TestBase { ...@@ -92,7 +92,7 @@ public class TestRandomMapOps extends TestBase {
for (; op < size; op++) { for (; op < size; op++) {
int k = r.nextInt(100); int k = r.nextInt(100);
byte[] v = new byte[r.nextInt(10) * 10]; byte[] v = new byte[r.nextInt(10) * 10];
int type = r.nextInt(13); int type = r.nextInt(12);
switch (type) { switch (type) {
case 0: case 0:
case 1: case 1:
...@@ -109,23 +109,19 @@ public class TestRandomMapOps extends TestBase { ...@@ -109,23 +109,19 @@ public class TestRandomMapOps extends TestBase {
map.remove(k); map.remove(k);
break; break;
case 6: case 6:
log(op, k, v, "s.store()");
s.store();
break;
case 7:
log(op, k, v, "s.compact(90)"); log(op, k, v, "s.compact(90)");
s.compact(90); s.compact(90);
break; break;
case 8: case 7:
log(op, k, v, "m.clear()"); log(op, k, v, "m.clear()");
m.clear(); m.clear();
map.clear(); map.clear();
break; break;
case 9: case 8:
log(op, k, v, "s.commit()"); log(op, k, v, "s.commit()");
s.commit(); s.commit();
break; break;
case 10: case 9:
log(op, k, v, "s.commit()"); log(op, k, v, "s.commit()");
s.commit(); s.commit();
log(op, k, v, "s.close()"); log(op, k, v, "s.close()");
...@@ -135,13 +131,13 @@ public class TestRandomMapOps extends TestBase { ...@@ -135,13 +131,13 @@ public class TestRandomMapOps extends TestBase {
log(op, k, v, "m = s.openMap(\"data\")"); log(op, k, v, "m = s.openMap(\"data\")");
m = s.openMap("data"); m = s.openMap("data");
break; break;
case 11: case 10:
log(op, k, v, "s.commit()"); log(op, k, v, "s.commit()");
s.commit(); s.commit();
log(op, k, v, "s.compactMoveChunks()"); log(op, k, v, "s.compactMoveChunks()");
s.compactMoveChunks(); s.compactMoveChunks();
break; break;
case 12: case 11:
log(op, k, v, "m.getKeyIndex({0})"); log(op, k, v, "m.getKeyIndex({0})");
ArrayList<Integer> keyList = new ArrayList<Integer>(map.keySet()); ArrayList<Integer> keyList = new ArrayList<Integer>(map.keySet());
int index = Collections.binarySearch(keyList, k, null); int index = Collections.binarySearch(keyList, k, null);
...@@ -165,13 +161,12 @@ public class TestRandomMapOps extends TestBase { ...@@ -165,13 +161,12 @@ public class TestRandomMapOps extends TestBase {
assertEquals(map.lastKey(), m.lastKey()); assertEquals(map.lastKey(), m.lastKey());
} }
} }
s.store();
s.close(); s.close();
} }
private static MVStore openStore(String fileName) { private static MVStore openStore(String fileName) {
MVStore s = new MVStore.Builder().fileName(fileName). MVStore s = new MVStore.Builder().fileName(fileName).
pageSplitSize(50).writeDelay(0).open(); pageSplitSize(50).autoCommitDisabled().open();
s.setRetentionTime(0); s.setRetentionTime(0);
return s; return s;
} }
......
...@@ -65,7 +65,7 @@ public class TestStreamStore extends TestBase { ...@@ -65,7 +65,7 @@ public class TestStreamStore extends TestBase {
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
streamStore.put(new RandomStream(size, i)); streamStore.put(new RandomStream(size, i));
} }
s.store(); s.commit();
MVMap<Long, byte[]> map = s.openMap("data"); MVMap<Long, byte[]> map = s.openMap("data");
assertTrue("size: " + map.size(), map.sizeAsLong() >= 100); assertTrue("size: " + map.size(), map.sizeAsLong() >= 100);
s.close(); s.close();
...@@ -77,7 +77,7 @@ public class TestStreamStore extends TestBase { ...@@ -77,7 +77,7 @@ public class TestStreamStore extends TestBase {
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
streamStore.put(new RandomStream(size, -i)); streamStore.put(new RandomStream(size, -i));
} }
s.store(); s.commit();
long readCount = s.getFileStore().getReadCount(); long readCount = s.getFileStore().getReadCount();
// the read count should be low because new blocks // the read count should be low because new blocks
// are appended at the end (not between existing blocks) // are appended at the end (not between existing blocks)
...@@ -92,7 +92,7 @@ public class TestStreamStore extends TestBase { ...@@ -92,7 +92,7 @@ public class TestStreamStore extends TestBase {
return new StreamStore(map) { return new StreamStore(map) {
@Override @Override
protected void onStore(int len) { protected void onStore(int len) {
if (s.getUnsavedPageCount() > s.getUnsavedPageCountMax() / 2) { if (s.getUnsavedPageCount() > s.getAutoCommitPageCount() / 2) {
s.commit(); s.commit();
} }
} }
...@@ -111,12 +111,11 @@ public class TestStreamStore extends TestBase { ...@@ -111,12 +111,11 @@ public class TestStreamStore extends TestBase {
@Override @Override
protected void onStore(int len) { protected void onStore(int len) {
count.incrementAndGet(); count.incrementAndGet();
s.store(); s.commit();
} }
}; };
long size = 1 * 1024 * 1024; long size = 1 * 1024 * 1024;
streamStore.put(new RandomStream(size, 0)); streamStore.put(new RandomStream(size, 0));
s.store();
s.close(); s.close();
assertEquals(4, count.get()); assertEquals(4, count.get());
} }
......
...@@ -86,7 +86,7 @@ public class TestTransactionStore extends TestBase { ...@@ -86,7 +86,7 @@ public class TestTransactionStore extends TestBase {
for (int i = 0; !stop; i++) { for (int i = 0; !stop; i++) {
state.set(i); state.set(i);
other.put(i, value); other.put(i, value);
store.store(); store.commit();
} }
} }
}; };
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论