提交 2ed0a04b authored 作者: Thomas Mueller's avatar Thomas Mueller

MVStore: the file format was changed slightly.

上级 b9633b12
...@@ -82,6 +82,12 @@ public class Chunk { ...@@ -82,6 +82,12 @@ public class Chunk {
* The last used map id. * The last used map id.
*/ */
public int mapId; public int mapId;
/**
* The predicted position of the next chunk.
*/
public long next;
public long nextSize;
Chunk(int id) { Chunk(int id) {
this.id = id; this.id = id;
...@@ -94,7 +100,7 @@ public class Chunk { ...@@ -94,7 +100,7 @@ public class Chunk {
* @param start the start of the chunk in the file * @param start the start of the chunk in the file
* @return the chunk * @return the chunk
*/ */
static Chunk fromHeader(ByteBuffer buff, long start) { static Chunk readChunkHeader(ByteBuffer buff, long start) {
int pos = buff.position(); int pos = buff.position();
byte[] data = new byte[Math.min(buff.remaining(), MAX_HEADER_LENGTH)]; byte[] data = new byte[Math.min(buff.remaining(), MAX_HEADER_LENGTH)];
buff.get(data); buff.get(data);
...@@ -120,7 +126,7 @@ public class Chunk { ...@@ -120,7 +126,7 @@ public class Chunk {
* *
* @param buff the target buffer * @param buff the target buffer
*/ */
void writeHeader(WriteBuffer buff, int minLength) { void writeChunkHeader(WriteBuffer buff, int minLength) {
long pos = buff.position(); long pos = buff.position();
buff.put(asString().getBytes(DataUtils.LATIN)); buff.put(asString().getBytes(DataUtils.LATIN));
while (buff.position() - pos < minLength - 1) { while (buff.position() - pos < minLength - 1) {
...@@ -152,7 +158,8 @@ public class Chunk { ...@@ -152,7 +158,8 @@ public class Chunk {
c.maxLenLive = DataUtils.readHexLong(map, "liveMax", c.maxLength); c.maxLenLive = DataUtils.readHexLong(map, "liveMax", c.maxLength);
c.metaRootPos = DataUtils.readHexLong(map, "root", 0); c.metaRootPos = DataUtils.readHexLong(map, "root", 0);
c.time = DataUtils.readHexLong(map, "time", 0); c.time = DataUtils.readHexLong(map, "time", 0);
c.version = DataUtils.readHexLong(map, "version", 0); c.version = DataUtils.readHexLong(map, "version", id);
c.next = DataUtils.readHexLong(map, "next", 0);
return c; return c;
} }
...@@ -188,12 +195,34 @@ public class Chunk { ...@@ -188,12 +195,34 @@ public class Chunk {
} }
DataUtils.appendMap(buff, "map", mapId); DataUtils.appendMap(buff, "map", mapId);
DataUtils.appendMap(buff, "max", maxLength); DataUtils.appendMap(buff, "max", maxLength);
if (next != 0) {
DataUtils.appendMap(buff, "next", next);
}
DataUtils.appendMap(buff, "pages", pageCount); DataUtils.appendMap(buff, "pages", pageCount);
DataUtils.appendMap(buff, "root", metaRootPos); DataUtils.appendMap(buff, "root", metaRootPos);
DataUtils.appendMap(buff, "time", time); DataUtils.appendMap(buff, "time", time);
DataUtils.appendMap(buff, "version", version); if (version != id) {
DataUtils.appendMap(buff, "version", version);
}
return buff.toString(); return buff.toString();
} }
byte[] getFooterBytes() {
StringBuilder buff = new StringBuilder();
DataUtils.appendMap(buff, "chunk", id);
DataUtils.appendMap(buff, "block", block);
if (version != id) {
DataUtils.appendMap(buff, "version", version);
}
byte[] bytes = buff.toString().getBytes(DataUtils.LATIN);
int checksum = DataUtils.getFletcher32(bytes, bytes.length / 2 * 2);
DataUtils.appendMap(buff, "fletcher", checksum);
while (buff.length() < MVStore.CHUNK_FOOTER_LENGTH - 1) {
buff.append(' ');
}
buff.append("\n");
return buff.toString().getBytes(DataUtils.LATIN);
}
@Override @Override
public String toString() { public String toString() {
......
...@@ -85,7 +85,7 @@ public class MVStoreTool { ...@@ -85,7 +85,7 @@ public class MVStoreTool {
continue; continue;
} }
block.position(0); block.position(0);
Chunk c = Chunk.fromHeader(block, pos); Chunk c = Chunk.readChunkHeader(block, pos);
int length = c.len * MVStore.BLOCK_SIZE; int length = c.len * MVStore.BLOCK_SIZE;
pw.println(" " + c.toString()); pw.println(" " + c.toString());
ByteBuffer chunk = ByteBuffer.allocate(length); ByteBuffer chunk = ByteBuffer.allocate(length);
......
...@@ -50,8 +50,7 @@ public class OffHeapStore extends FileStore { ...@@ -50,8 +50,7 @@ public class OffHeapStore extends FileStore {
freeSpace.free(pos, length); freeSpace.free(pos, length);
ByteBuffer buff = memory.remove(pos); ByteBuffer buff = memory.remove(pos);
if (buff == null) { if (buff == null) {
throw DataUtils.newIllegalStateException(DataUtils.ERROR_READING_FAILED, // nothing was written (just allocated)
"Could not find entry at position {0}", pos);
} else if (buff.remaining() != length) { } else if (buff.remaining() != length) {
throw DataUtils.newIllegalStateException(DataUtils.ERROR_READING_FAILED, throw DataUtils.newIllegalStateException(DataUtils.ERROR_READING_FAILED,
"Partial remove is not supported at position {0}", pos); "Partial remove is not supported at position {0}", pos);
...@@ -105,6 +104,7 @@ public class OffHeapStore extends FileStore { ...@@ -105,6 +104,7 @@ public class OffHeapStore extends FileStore {
memory.clear(); memory.clear();
return; return;
} }
fileSize = size;
for (Iterator<Long> it = memory.keySet().iterator(); it.hasNext();) { for (Iterator<Long> it = memory.keySet().iterator(); it.hasNext();) {
long pos = it.next(); long pos = it.next();
if (pos < size) { if (pos < size) {
......
...@@ -187,7 +187,7 @@ public class Page { ...@@ -187,7 +187,7 @@ public class Page {
int length = maxLength; int length = maxLength;
if (length < 0) { if (length < 0) {
throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT, throw DataUtils.newIllegalStateException(DataUtils.ERROR_FILE_CORRUPT,
"Illegal page length {0} reading at {1}; file size {1} ", length, filePos, fileSize); "Illegal page length {0} reading at {1}; file size {2} ", length, filePos, fileSize);
} }
buff = fileStore.readFully(filePos, length); buff = fileStore.readFully(filePos, length);
Page p = new Page(map, 0); Page p = new Page(map, 0);
......
...@@ -182,7 +182,7 @@ public class MVTableEngine implements TableEngine { ...@@ -182,7 +182,7 @@ public class MVTableEngine implements TableEngine {
if (s == null || s.isReadOnly()) { if (s == null || s.isReadOnly()) {
return; return;
} }
if (!store.compact(50)) { if (!store.compact(50, 1024 * 1024)) {
store.commit(); store.commit();
} }
} }
...@@ -288,7 +288,7 @@ public class MVTableEngine implements TableEngine { ...@@ -288,7 +288,7 @@ public class MVTableEngine implements TableEngine {
public void compactFile(long maxCompactTime) { public void compactFile(long maxCompactTime) {
store.setRetentionTime(0); store.setRetentionTime(0);
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
while (store.compact(99)) { while (store.compact(99, 16 * 1024)) {
store.sync(); store.sync();
long time = System.currentTimeMillis() - start; long time = System.currentTimeMillis() - start;
if (time > maxCompactTime) { if (time > maxCompactTime) {
...@@ -311,7 +311,7 @@ public class MVTableEngine implements TableEngine { ...@@ -311,7 +311,7 @@ public class MVTableEngine implements TableEngine {
if (!store.getFileStore().isReadOnly()) { if (!store.getFileStore().isReadOnly()) {
transactionStore.close(); transactionStore.close();
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
while (store.compact(90)) { while (store.compact(90, 32 * 1024)) {
long time = System.currentTimeMillis() - start; long time = System.currentTimeMillis() - start;
if (time > maxCompactTime) { if (time > maxCompactTime) {
break; break;
......
...@@ -69,8 +69,8 @@ public class TestBenchmark extends TestBase { ...@@ -69,8 +69,8 @@ public class TestBenchmark extends TestBase {
for (int i = 0; i < rowCount; i++) { for (int i = 0; i < rowCount; i++) {
prep.setInt(1, i); prep.setInt(1, i);
prep.setInt(2, i); // prep.setInt(2, i);
// prep.setInt(2, r.nextInt()); prep.setInt(2, r.nextInt());
prep.execute(); prep.execute();
if (i % 10000 == 0) { if (i % 10000 == 0) {
conn.commit(); conn.commit();
......
...@@ -109,7 +109,7 @@ public class TestKillProcessWhileWriting extends TestBase { ...@@ -109,7 +109,7 @@ public class TestKillProcessWhileWriting extends TestBase {
s.commit(); s.commit();
break; break;
case 7: case 7:
s.compact(80); s.compact(80, 1024);
break; break;
case 8: case 8:
m.clear(); m.clear();
......
...@@ -192,7 +192,7 @@ public class TestMVStore extends TestBase { ...@@ -192,7 +192,7 @@ public class TestMVStore extends TestBase {
map.put(i, "Hello " + i); map.put(i, "Hello " + i);
s.commit(); s.commit();
} }
assertTrue(1000 < offHeap.getWriteCount()); assertTrue(offHeap.getWriteCount() > 1000);
s.close(); s.close();
s = new MVStore.Builder(). s = new MVStore.Builder().
...@@ -212,6 +212,7 @@ public class TestMVStore extends TestBase { ...@@ -212,6 +212,7 @@ public class TestMVStore extends TestBase {
encryptionKey("007".toCharArray()). encryptionKey("007".toCharArray()).
fileName(fileName). fileName(fileName).
open(); open();
s.setRetentionTime(Integer.MAX_VALUE);
Map<String, Object> header = s.getStoreHeader(); Map<String, Object> header = s.getStoreHeader();
assertEquals("1", header.get("format").toString()); assertEquals("1", header.get("format").toString());
header.put("formatRead", "1"); header.put("formatRead", "1");
...@@ -528,19 +529,20 @@ public class TestMVStore extends TestBase { ...@@ -528,19 +529,20 @@ public class TestMVStore extends TestBase {
MVStore s; MVStore s;
MVMap<Integer, Integer> m; MVMap<Integer, Integer> m;
s = openStore(fileName); s = openStore(fileName);
s.setRetentionTime(Integer.MAX_VALUE);
m = s.openMap("test"); m = s.openMap("test");
m.put(1, 1); m.put(1, 1);
Map<String, Object> header = s.getStoreHeader(); Map<String, Object> header = s.getStoreHeader();
int format = Integer.parseInt(header.get("format").toString()); int format = Integer.parseInt(header.get("format").toString());
assertEquals(1, format); assertEquals(1, format);
header.put("format", Integer.toString(format + 1)); header.put("format", Integer.toString(format + 1));
// ensure the file header is overwritten for (int i = 0; i < 10; i++) {
s.commit(); if (i > 5) {
m.put(1, 10); s.setRetentionTime(0);
s.commit(); }
m.put(1, 20); m.put(10, 100 * i);
s.setRetentionTime(0); s.commit();
s.commit(); }
s.close(); s.close();
try { try {
openStore(fileName).close(); openStore(fileName).close();
...@@ -662,6 +664,7 @@ public class TestMVStore extends TestBase { ...@@ -662,6 +664,7 @@ public class TestMVStore extends TestBase {
private void testFileHeader() { private void testFileHeader() {
String fileName = getBaseDir() + "/testFileHeader.h3"; String fileName = getBaseDir() + "/testFileHeader.h3";
MVStore s = openStore(fileName); MVStore s = openStore(fileName);
s.setRetentionTime(Integer.MAX_VALUE);
long time = System.currentTimeMillis(); long time = System.currentTimeMillis();
Map<String, Object> m = s.getStoreHeader(); Map<String, Object> m = s.getStoreHeader();
assertEquals("1", m.get("format").toString()); assertEquals("1", m.get("format").toString());
...@@ -671,12 +674,13 @@ public class TestMVStore extends TestBase { ...@@ -671,12 +674,13 @@ public class TestMVStore extends TestBase {
MVMap<Integer, Integer> map = s.openMap("test"); MVMap<Integer, Integer> map = s.openMap("test");
map.put(10, 100); map.put(10, 100);
// ensure the file header is overwritten // ensure the file header is overwritten
s.commit(); for (int i = 0; i < 10; i++) {
map.put(10, 110); if (i > 5) {
s.commit(); s.setRetentionTime(0);
map.put(1, 120); }
s.setRetentionTime(0); map.put(10, 110);
s.commit(); s.commit();
}
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
assertEquals("123", s.getStoreHeader().get("test").toString()); assertEquals("123", s.getStoreHeader().get("test").toString());
...@@ -701,7 +705,7 @@ public class TestMVStore extends TestBase { ...@@ -701,7 +705,7 @@ public class TestMVStore extends TestBase {
map = s.openMap("test" + i); map = s.openMap("test" + i);
s.removeMap(map); s.removeMap(map);
s.commit(); s.commit();
s.compact(100); s.compact(100, 1);
if (fs.getFile().size() <= size) { if (fs.getFile().size() <= size) {
break; break;
} }
...@@ -1058,6 +1062,7 @@ public class TestMVStore extends TestBase { ...@@ -1058,6 +1062,7 @@ public class TestMVStore extends TestBase {
FileUtils.delete(fileName); FileUtils.delete(fileName);
MVStore s; MVStore s;
s = openStore(fileName); s = openStore(fileName);
s.setRetentionTime(Integer.MAX_VALUE);
MVMap<String, String> m; MVMap<String, String> m;
m = s.openMap("data"); m = s.openMap("data");
long first = s.getCurrentVersion(); long first = s.getCurrentVersion();
...@@ -1121,16 +1126,24 @@ public class TestMVStore extends TestBase { ...@@ -1121,16 +1126,24 @@ public class TestMVStore extends TestBase {
MVMap<Integer, String> m; MVMap<Integer, String> m;
s = openStore(fileName); s = openStore(fileName);
m = s.openMap("data"); m = s.openMap("data");
for (int i = 0; i < 1000; i++) { String data = new String(new char[10000]).replace((char) 0, 'x');
m.put(i, "Hello World"); for (int i = 1; i < 10; i++) {
m.put(i, data);
s.commit();
} }
s.close(); s.close();
long len = FileUtils.size(fileName); long len = FileUtils.size(fileName);
s = openStore(fileName); s = openStore(fileName);
s.setRetentionTime(0); s.setRetentionTime(0);
// remove 50%
m = s.openMap("data"); m = s.openMap("data");
m.clear(); for (int i = 0; i < 10; i += 2) {
s.compact(100); m.remove(i);
}
s.commit();
assertTrue(s.compact(100, 1));
assertTrue(s.compact(100, 1));
assertTrue(s.compact(100, 1));
s.close(); s.close();
long len2 = FileUtils.size(fileName); long len2 = FileUtils.size(fileName);
assertTrue("len2: " + len2 + " len: " + len, len2 < len); assertTrue("len2: " + len2 + " len: " + len, len2 < len);
...@@ -1160,7 +1173,7 @@ public class TestMVStore extends TestBase { ...@@ -1160,7 +1173,7 @@ public class TestMVStore extends TestBase {
s.commit(); s.commit();
// ensure only nodes are read, but not leaves // ensure only nodes are read, but not leaves
assertEquals(40, s.getFileStore().getReadCount()); assertEquals(40, s.getFileStore().getReadCount());
assertEquals(1, s.getFileStore().getWriteCount()); assertTrue(s.getFileStore().getWriteCount() < 5);
s.close(); s.close();
} }
...@@ -1209,6 +1222,7 @@ public class TestMVStore extends TestBase { ...@@ -1209,6 +1222,7 @@ public class TestMVStore extends TestBase {
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
s.setRetentionTime(45000);
assertEquals(2, s.getCurrentVersion()); assertEquals(2, s.getCurrentVersion());
meta = s.getMetaMap(); meta = s.getMetaMap();
m = s.openMap("data"); m = s.openMap("data");
...@@ -1231,6 +1245,7 @@ public class TestMVStore extends TestBase { ...@@ -1231,6 +1245,7 @@ public class TestMVStore extends TestBase {
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
s.setRetentionTime(45000);
assertEquals(2, s.getCurrentVersion()); assertEquals(2, s.getCurrentVersion());
meta = s.getMetaMap(); meta = s.getMetaMap();
assertTrue(meta.get("name.data") != null); assertTrue(meta.get("name.data") != null);
...@@ -1248,12 +1263,14 @@ public class TestMVStore extends TestBase { ...@@ -1248,12 +1263,14 @@ public class TestMVStore extends TestBase {
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
s.setRetentionTime(45000);
assertEquals(3, s.getCurrentVersion()); assertEquals(3, s.getCurrentVersion());
m = s.openMap("data"); m = s.openMap("data");
m.put("1", "Hi"); m.put("1", "Hi");
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
s.setRetentionTime(45000);
m = s.openMap("data"); m = s.openMap("data");
assertEquals("Hi", m.get("1")); assertEquals("Hi", m.get("1"));
s.rollbackTo(v3); s.rollbackTo(v3);
...@@ -1261,6 +1278,7 @@ public class TestMVStore extends TestBase { ...@@ -1261,6 +1278,7 @@ public class TestMVStore extends TestBase {
s.close(); s.close();
s = openStore(fileName); s = openStore(fileName);
s.setRetentionTime(45000);
m = s.openMap("data"); m = s.openMap("data");
assertEquals("Hallo", m.get("1")); assertEquals("Hallo", m.get("1"));
s.close(); s.close();
...@@ -1310,6 +1328,7 @@ public class TestMVStore extends TestBase { ...@@ -1310,6 +1328,7 @@ public class TestMVStore extends TestBase {
String fileName = getBaseDir() + "/testMeta.h3"; String fileName = getBaseDir() + "/testMeta.h3";
FileUtils.delete(fileName); FileUtils.delete(fileName);
MVStore s = openStore(fileName); MVStore s = openStore(fileName);
s.setRetentionTime(Integer.MAX_VALUE);
MVMap<String, String> m = s.getMetaMap(); MVMap<String, String> m = s.getMetaMap();
assertEquals("[]", s.getMapNames().toString()); assertEquals("[]", s.getMapNames().toString());
MVMap<String, String> data = s.openMap("data"); MVMap<String, String> data = s.openMap("data");
...@@ -1480,9 +1499,8 @@ public class TestMVStore extends TestBase { ...@@ -1480,9 +1499,8 @@ public class TestMVStore extends TestBase {
chunkCount1++; chunkCount1++;
} }
} }
s.compact(80, 1);
assertTrue(s.compact(80)); s.compact(80, 1);
assertTrue(s.compact(80));
int chunkCount2 = 0; int chunkCount2 = 0;
for (String k : meta.keySet()) { for (String k : meta.keySet()) {
...@@ -1493,8 +1511,8 @@ public class TestMVStore extends TestBase { ...@@ -1493,8 +1511,8 @@ public class TestMVStore extends TestBase {
assertTrue(chunkCount2 >= chunkCount1); assertTrue(chunkCount2 >= chunkCount1);
m = s.openMap("data"); m = s.openMap("data");
assertTrue(s.compact(80)); assertTrue(s.compact(80, 16 * 1024));
assertTrue(s.compact(80)); assertTrue(s.compact(80, 1024));
int chunkCount3 = 0; int chunkCount3 = 0;
for (String k : meta.keySet()) { for (String k : meta.keySet()) {
...@@ -1522,7 +1540,7 @@ public class TestMVStore extends TestBase { ...@@ -1522,7 +1540,7 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
m.put(j + i, "Hello " + j); m.put(j + i, "Hello " + j);
} }
s.compact(80); s.compact(80, 1024);
s.close(); s.close();
long len = FileUtils.size(fileName); long len = FileUtils.size(fileName);
// System.out.println(" len:" + len); // System.out.println(" len:" + len);
...@@ -1539,13 +1557,13 @@ public class TestMVStore extends TestBase { ...@@ -1539,13 +1557,13 @@ public class TestMVStore extends TestBase {
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
m.remove(i); m.remove(i);
} }
s.compact(80); s.compact(80, 1024);
s.close(); s.close();
// len = FileUtils.size(fileName); // len = FileUtils.size(fileName);
// System.out.println("len1: " + len); // System.out.println("len1: " + len);
s = openStore(fileName); s = openStore(fileName);
m = s.openMap("data"); m = s.openMap("data");
s.compact(80); s.compact(80, 1024);
s.close(); s.close();
// len = FileUtils.size(fileName); // len = FileUtils.size(fileName);
// System.out.println("len2: " + len); // System.out.println("len2: " + len);
......
...@@ -45,16 +45,17 @@ public class TestRandomMapOps extends TestBase { ...@@ -45,16 +45,17 @@ public class TestRandomMapOps extends TestBase {
testMap("memFS:randomOps.h3"); testMap("memFS:randomOps.h3");
} }
private void testMap(String fileName) { private void testMap(String fileName) throws Exception {
this.fileName = fileName; this.fileName = fileName;
int best = Integer.MAX_VALUE; int best = Integer.MAX_VALUE;
int bestSeed = 0; int bestSeed = 0;
Throwable failException = null; Throwable failException = null;
int size = getSize(100, 1000);
for (seed = 0; seed < 100; seed++) { for (seed = 0; seed < 100; seed++) {
FileUtils.delete(fileName); FileUtils.delete(fileName);
Throwable ex = null; Throwable ex = null;
try { try {
testCase(); testOps(size);
continue; continue;
} catch (Exception e) { } catch (Exception e) {
ex = e; ex = e;
...@@ -65,6 +66,7 @@ public class TestRandomMapOps extends TestBase { ...@@ -65,6 +66,7 @@ public class TestRandomMapOps extends TestBase {
trace(seed); trace(seed);
bestSeed = seed; bestSeed = seed;
best = op; best = op;
size = best;
failException = ex; failException = ex;
// System.out.println("seed:" + seed + " op:" + op); // System.out.println("seed:" + seed + " op:" + op);
} }
...@@ -75,7 +77,7 @@ public class TestRandomMapOps extends TestBase { ...@@ -75,7 +77,7 @@ public class TestRandomMapOps extends TestBase {
} }
} }
private void testCase() throws Exception { private void testOps(int size) throws Exception {
FileUtils.delete(fileName); FileUtils.delete(fileName);
MVStore s; MVStore s;
s = openStore(fileName); s = openStore(fileName);
...@@ -87,7 +89,6 @@ public class TestRandomMapOps extends TestBase { ...@@ -87,7 +89,6 @@ public class TestRandomMapOps extends TestBase {
} }
Random r = new Random(seed); Random r = new Random(seed);
op = 0; op = 0;
int size = getSize(100, 1000);
TreeMap<Integer, byte[]> map = new TreeMap<Integer, byte[]>(); TreeMap<Integer, byte[]> map = new TreeMap<Integer, byte[]>();
for (; op < size; op++) { for (; op < size; op++) {
int k = r.nextInt(100); int k = r.nextInt(100);
...@@ -109,8 +110,8 @@ public class TestRandomMapOps extends TestBase { ...@@ -109,8 +110,8 @@ public class TestRandomMapOps extends TestBase {
map.remove(k); map.remove(k);
break; break;
case 6: case 6:
log(op, k, v, "s.compact(90)"); log(op, k, v, "s.compact(90, 1024)");
s.compact(90); s.compact(90, 1024);
break; break;
case 7: case 7:
log(op, k, v, "m.clear()"); log(op, k, v, "m.clear()");
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论