MVTableEngine: improve tests

h2/src/docsrc/html/mvstore.html

@@ -383,8 +383,13 @@ meaning outside of the regular garbage collected heap. This allows to use very l
 stores without having to increase the JVM heap (which would increase Java garbage collection
 pauses a lot). Memory is allocated using <code>ByteBuffer.allocateDirect</code>.
 One chunk is allocated at a time (each chunk is usually a few MB large), so that
-allocation cost is low.
+allocation cost is low. To use the off-heap storage, call:
 </p>
+<pre>
+OffHeapStore offHeap = new OffHeapStore();
+MVStore s = new MVStore.Builder().
+        fileStore(offHeap).open();
+</pre>
 
 <h3 id="fileSystem">File System Abstraction, File Locking and Online Backup</h3>
 <p>

h2/src/main/org/h2/mvstore/MVStore.java

@@ -110,6 +110,8 @@ MVStore:
 - add new feature to file systems that avoid copying data
     (reads should return a ByteBuffer, not write into one)
 - do we need to store a dummy chunk entry in the chunk itself?
+    currently yes, as some fields are not set in the chunk header
+- off-heap LIRS cache (the LIRS cache should use a map factory)
 
 */
 

h2/src/main/org/h2/store/fs/FilePathMem.java

@@ -30,6 +30,7 @@ import org.h2.util.New;
 public class FilePathMem extends FilePath {
 
     private static final TreeMap<String, FileMemData> MEMORY_FILES = new TreeMap<String, FileMemData>();
+    private static final FileMemData DIRECTORY = new FileMemData("", false);
 
     @Override
     public FilePathMem getPath(String path) {
@@ -90,7 +91,9 @@ public class FilePathMem extends FilePath {
         synchronized (MEMORY_FILES) {
             for (String n : MEMORY_FILES.tailMap(name).keySet()) {
                 if (n.startsWith(name)) {
+                    if (!n.equals(name) && n.indexOf('/', name.length() + 1) < 0) {
                         list.add(getPath(n));
+                    }
                 } else {
                     break;
                 }
@@ -120,10 +123,9 @@ public class FilePathMem extends FilePath {
         if (isRoot()) {
             return true;
         }
-        // TODO in memory file system currently
-        // does not really support directories
         synchronized (MEMORY_FILES) {
-            return MEMORY_FILES.get(name) == null;
+            FileMemData d = MEMORY_FILES.get(name);
+            return d == DIRECTORY;
         }
     }
 
@@ -145,10 +147,12 @@ public class FilePathMem extends FilePath {
 
     @Override
     public void createDirectory() {
-        if (exists() && isDirectory()) {
+        if (exists()) {
             throw DbException.get(ErrorCode.FILE_CREATION_FAILED_1, name + " (a file with this name already exists)");
         }
-        // TODO directories are not really supported
+        synchronized (MEMORY_FILES) {
+            MEMORY_FILES.put(name, DIRECTORY);
+        }
     }
 
     @Override
@@ -174,6 +178,9 @@ public class FilePathMem extends FilePath {
     private FileMemData getMemoryFile() {
         synchronized (MEMORY_FILES) {
             FileMemData m = MEMORY_FILES.get(name);
+            if (m == DIRECTORY) {
+                throw DbException.get(ErrorCode.FILE_CREATION_FAILED_1, name + " (a directory with this name already exists)");
+            }
             if (m == null) {
                 m = new FileMemData(name, compressed());
                 MEMORY_FILES.put(name, m);

h2/src/main/org/h2/store/fs/FilePathRec.java

@@ -154,6 +154,11 @@ class FileRec extends FileBase {
         return channel.read(dst);
     }
 
+    @Override
+    public int read(ByteBuffer dst, long position) throws IOException {
+        return channel.read(dst, position);
+    }
+
     @Override
     public FileChannel position(long pos) throws IOException {
         channel.position(pos);
@@ -186,6 +191,20 @@ class FileRec extends FileBase {
         return result;
     }
     
+    @Override
+    public int write(ByteBuffer src, long position) throws IOException {
+        byte[] buff = src.array();
+        int len = src.remaining();
+        if (src.position() != 0 || len != buff.length) {
+            byte[] b = new byte[len];
+            System.arraycopy(buff, src.arrayOffset() + src.position(), b, 0, len);
+            buff = b;
+        }
+        int result = channel.write(src, position);
+        rec.log(Recorder.WRITE, name, buff, position);
+        return result;
+    }
+
     @Override
     public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
         return channel.tryLock(position, size, shared);

h2/src/test/org/h2/test/db/TestFullText.java

@@ -57,6 +57,7 @@ public class TestFullText extends TestBase {
         testPerformance(false);
         testReopen(false);
         testDropIndex(false);
+        if (!config.reopen) {
             try {
                 Class.forName(LUCENE_FULLTEXT_CLASS_NAME);
                 testCreateDropLucene();
@@ -77,6 +78,7 @@ public class TestFullText extends TestBase {
                 // ok
             }
             FullText.closeAll();
+        }
         deleteDb("fullText");
         deleteDb("fullTextReopen");
     }

h2/src/test/org/h2/test/store/TestKillProcessWhileWriting.java

@@ -9,7 +9,6 @@ import java.util.Random;
 
 import org.h2.mvstore.MVMap;
 import org.h2.mvstore.MVStore;
-import org.h2.store.fs.FilePathCrypt;
 import org.h2.store.fs.FileUtils;
 import org.h2.test.TestBase;
 import org.h2.test.utils.FilePathUnstable;
@@ -29,28 +28,33 @@ public class TestKillProcessWhileWriting extends TestBase {
      * @param a ignored
      */
     public static void main(String... a) throws Exception {
-        TestBase.createCaller().init().test();
+        TestBase test = TestBase.createCaller().init();
+        test.config.big = true;
+        test.test();
     }
 
     @Override
     public void test() throws Exception {
         fs = FilePathUnstable.register();
+        fs.setPartialWrites(false);
         test("unstable:memFS:killProcess.h3");
 
-        int todo;
-        // need to test with a file system splits writes into blocks of 4 KB
-        FilePathCrypt.register();
-        test("unstable:crypt:0007:memFS:killProcess.h3");
+        if (config.big) { 
+            fs.setPartialWrites(true);
+            test("unstable:memFS:killProcess.h3");
+        }
     }
 
     private void test(String fileName) throws Exception {
         for (seed = 0; seed < 10; seed++) {
             this.fileName = fileName;
+            fs.setSeed(seed);
             FileUtils.delete(fileName);
             test(Integer.MAX_VALUE);
             int max = Integer.MAX_VALUE - fs.getDiskFullCount() + 10;
             assertTrue("" + (max - 10), max > 0);
             for (int i = 0; i < max; i++) {
+                fs.setSeed(seed);
                 test(i);
             }
         }

h2/src/test/org/h2/test/unit/TestReopen.java

@@ -71,7 +71,8 @@ public class TestReopen extends TestBase implements Recorder {
         if (op != Recorder.WRITE && op != Recorder.TRUNCATE) {
             return;
         }
-        if (!fileName.endsWith(Constants.SUFFIX_PAGE_FILE)) {
+        if (!fileName.endsWith(Constants.SUFFIX_PAGE_FILE) && 
+                !fileName.endsWith(Constants.SUFFIX_MV_FILE)) {
             return;
         }
         if (testing) {
@@ -98,14 +99,22 @@ public class TestReopen extends TestBase implements Recorder {
         System.out.println("+ write #" + writeCount + " verify #" + verifyCount);
 
         try {
+            if (fileName.endsWith(Constants.SUFFIX_PAGE_FILE)) {
                 IOUtils.copyFiles(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
+            } else {
+                IOUtils.copyFiles(fileName, testDatabase + Constants.SUFFIX_MV_FILE);
+            }
             verifyCount++;
             // avoid using the Engine class to avoid deadlocks
             Properties p = new Properties();
             String userName =  getUser();
             p.setProperty("user", userName);
             p.setProperty("password", getPassword());
-            ConnectionInfo ci = new ConnectionInfo("jdbc:h2:" + testDatabase + ";FILE_LOCK=NO;TRACE_LEVEL_FILE=0", p);
+            String url = "jdbc:h2:" + testDatabase + ";FILE_LOCK=NO;TRACE_LEVEL_FILE=0";
+            if (config.mvStore) {
+                url += ";MV_STORE=TRUE";
+            }
+            ConnectionInfo ci = new ConnectionInfo(url, p);
             Database database = new Database(ci, null);
             // close the database
             Session session = database.getSystemSession();
@@ -145,10 +154,18 @@ public class TestReopen extends TestBase implements Recorder {
         }
         testDatabase += "X";
         try {
+            if (fileName.endsWith(Constants.SUFFIX_PAGE_FILE)) {
                 IOUtils.copyFiles(fileName, testDatabase + Constants.SUFFIX_PAGE_FILE);
+            } else {
+                IOUtils.copyFiles(fileName, testDatabase + Constants.SUFFIX_MV_FILE);
+            }
             // avoid using the Engine class to avoid deadlocks
             Properties p = new Properties();
-            ConnectionInfo ci = new ConnectionInfo("jdbc:h2:" + testDatabase + ";FILE_LOCK=NO", p);
+            String url = "jdbc:h2:" + testDatabase + ";FILE_LOCK=NO";
+            if (config.mvStore) {
+                url += ";MV_STORE=TRUE";
+            }
+            ConnectionInfo ci = new ConnectionInfo(url, p);
             Database database = new Database(ci, null);
             // close the database
             database.removeSession(null);

h2/src/test/org/h2/test/utils/FilePathUnstable.java

@@ -13,6 +13,8 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.util.List;
+import java.util.Random;
+
 import org.h2.store.fs.FileBase;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FilePathWrapper;
@@ -29,6 +31,10 @@ public class FilePathUnstable extends FilePathWrapper {
 
     private static int diskFullOffCount;
     
+    private static boolean partialWrites;
+    
+    private static Random random = new Random(1);
+
     /**
      * Register the file system.
      *
@@ -39,6 +45,42 @@ public class FilePathUnstable extends FilePathWrapper {
         return INSTANCE;
     }
     
+    /**
+     * Whether partial writes are possible (writing only part of the data).
+     * 
+     * @param partialWrites true to enable
+     */
+    public void setPartialWrites(boolean partialWrites) {
+        FilePathUnstable.partialWrites = partialWrites;
+    }
+    
+    boolean getPartialWrites() {
+        return partialWrites;
+    }
+    
+    /**
+     * Set the random seed.
+     * 
+     * @param seed the new seed
+     */
+    public void setSeed(long seed) {
+        random.setSeed(seed);
+    }
+    
+    /**
+     * Get a buffer with a subset (the head) of the data of the source buffer.
+     * 
+     * @param src the source buffer
+     * @return a buffer with a subset of the data
+     */
+    ByteBuffer getRandomSubset(ByteBuffer src) {
+        int len = src.remaining();
+        len = Math.min(4096, Math.min(len, 1 + random.nextInt(len)));
+        ByteBuffer temp = ByteBuffer.allocate(len);
+        src.get(temp.array());
+        return temp;
+    }
+
     /**
      * Check if the simulated problem occurred.
      * This call will decrement the countdown.
@@ -52,7 +94,7 @@ public class FilePathUnstable extends FilePathWrapper {
         if (--diskFullOffCount > 0) {
             return;
         }
-        if (diskFullOffCount >= -4) {
+        if (diskFullOffCount >= -1) {
             diskFullOffCount--;
             throw DISK_FULL;
         }
@@ -231,12 +273,18 @@ class FileUnstable extends FileBase {
     @Override
     public int write(ByteBuffer src) throws IOException {
         checkError();
+        if (file.getPartialWrites()) {
+            return channel.write(file.getRandomSubset(src));
+        }
         return channel.write(src);
     }
     
     @Override
     public int write(ByteBuffer src, long position) throws IOException {
         checkError();
+        if (file.getPartialWrites()) {
+            return channel.write(file.getRandomSubset(src), position);
+        }
         return channel.write(src, position);
     }