Experimental off-heap memory storage engine "nioMemFS:" and "nioMemLZF:",

suggestion from markaddleman@gmail.com

h2/src/docsrc/html/changelog.html

@@ -60,6 +60,8 @@ Change Log
 </li><li>Supporting dropping an index for Lucene full-text indexes.
 </li><li>Optimized performance for SELECT ... ORDER BY X LIMIT Y OFFSET Z
     queries for in-memory databases using partial sort (by Sergi Vladykin).
+</li><li>Experimental off-heap memory storage engine "nioMemFS:" and "nioMemLZF:",
+    suggestion from markaddleman@gmail.com
 </li></ul>
 
 <h2>Version 1.3.170 (2012-11-30)</h2>

h2/src/main/org/h2/compress/CompressLZF.java

@@ -49,6 +49,8 @@
 
 package org.h2.compress;
 
+import java.nio.ByteBuffer;
+
 /**
  * <p>
  * This class implements the LZF lossless data compression algorithm. LZF is a
@@ -122,6 +124,13 @@ public final class CompressLZF implements Compressor {
         return (in[inPos] << 8) | (in[inPos + 1] & 255);
     }
 
+    /**
+     * Return byte with lower 2 bytes being byte at index, then index+1.
+     */
+    private static int first(ByteBuffer in, int inPos) {
+        return (in.get(inPos) << 8) | (in.get(inPos + 1) & 255);
+    }
+    
     /**
      * Shift v 1 byte left, add value at index inPos+2.
      */
@@ -129,6 +138,13 @@ public final class CompressLZF implements Compressor {
         return (v << 8) | (in[inPos + 2] & 255);
     }
 
+    /**
+     * Shift v 1 byte left, add value at index inPos+2.
+     */
+    private static int next(int v, ByteBuffer in, int inPos) {
+        return (v << 8) | (in.get(inPos + 2) & 255);
+    }
+    
     /**
      * Compute the address in the hash table.
      */
@@ -235,6 +251,105 @@ public final class CompressLZF implements Compressor {
         return outPos;
     }
 
+    public int compress(ByteBuffer in, int inLen, byte[] out, int outPos) {
+        int inPos = 0;
+        if (cachedHashTable == null) {
+            cachedHashTable = new int[HASH_SIZE];
+        }
+        int[] hashTab = cachedHashTable;
+        int literals = 0;
+        outPos++;
+        int future = first(in, 0);
+        while (inPos < inLen - 4) {
+            byte p2 = in.get(inPos + 2);
+            // next
+            future = (future << 8) + (p2 & 255);
+            int off = hash(future);
+            int ref = hashTab[off];
+            hashTab[off] = inPos;
+            // if (ref < inPos
+            //       && ref > 0
+            //       && (off = inPos - ref - 1) < MAX_OFF
+            //       && in[ref + 2] == p2
+            //       && (((in[ref] & 255) << 8) | (in[ref + 1] & 255)) ==
+            //           ((future >> 8) & 0xffff)) {
+            if (ref < inPos
+                        && ref > 0
+                        && (off = inPos - ref - 1) < MAX_OFF
+                        && in.get(ref + 2) == p2
+                        && in.get(ref + 1) == (byte) (future >> 8)
+                        && in.get(ref) == (byte) (future >> 16)) {
+                // match
+                int maxLen = inLen - inPos - 2;
+                if (maxLen > MAX_REF) {
+                    maxLen = MAX_REF;
+                }
+                if (literals == 0) {
+                    // multiple back-references,
+                    // so there is no literal run control byte
+                    outPos--;
+                } else {
+                    // set the control byte at the start of the literal run
+                    // to store the number of literals
+                    out[outPos - literals - 1] = (byte) (literals - 1);
+                    literals = 0;
+                }
+                int len = 3;
+                while (len < maxLen && in.get(ref + len) == in.get(inPos + len)) {
+                    len++;
+                }
+                len -= 2;
+                if (len < 7) {
+                    out[outPos++] = (byte) ((off >> 8) + (len << 5));
+                } else {
+                    out[outPos++] = (byte) ((off >> 8) + (7 << 5));
+                    out[outPos++] = (byte) (len - 7);
+                }
+                out[outPos++] = (byte) off;
+                // move one byte forward to allow for a literal run control byte
+                outPos++;
+                inPos += len;
+                // rebuild the future, and store the last bytes to the hashtable.
+                // Storing hashes of the last bytes in back-reference improves
+                // the compression ratio and only reduces speed slightly.
+                future = first(in, inPos);
+                future = next(future, in, inPos);
+                hashTab[hash(future)] = inPos++;
+                future = next(future, in, inPos);
+                hashTab[hash(future)] = inPos++;
+            } else {
+                // copy one byte from input to output as part of literal
+                out[outPos++] = in.get(inPos++);
+                literals++;
+                // at the end of this literal chunk, write the length
+                // to the control byte and start a new chunk
+                if (literals == MAX_LITERAL) {
+                    out[outPos - literals - 1] = (byte) (literals - 1);
+                    literals = 0;
+                    // move ahead one byte to allow for the
+                    // literal run control byte
+                    outPos++;
+                }
+            }
+        }
+        // write the remaining few bytes as literals
+        while (inPos < inLen) {
+            out[outPos++] = in.get(inPos++);
+            literals++;
+            if (literals == MAX_LITERAL) {
+                out[outPos - literals - 1] = (byte) (literals - 1);
+                literals = 0;
+                outPos++;
+            }
+        }
+        // writes the final literal run length to the control byte
+        out[outPos - literals - 1] = (byte) (literals - 1);
+        if (literals == 0) {
+            outPos--;
+        }
+        return outPos;
+    }
+    
     public void expand(byte[] in, int inPos, int inLen, byte[] out, int outPos, int outLen) {
         // if ((inPos | outPos | outLen) < 0) {
         if (inPos < 0 || outPos < 0 || outLen < 0) {
@@ -282,6 +397,53 @@ public final class CompressLZF implements Compressor {
         } while (outPos < outLen);
     }
 
+    public void expand(ByteBuffer in, int inPos, int inLen, ByteBuffer out, int outPos, int outLen) {
+        // if ((inPos | outPos | outLen) < 0) {
+        if (inPos < 0 || outPos < 0 || outLen < 0) {
+            throw new IllegalArgumentException();
+        }
+        do {
+            int ctrl = in.get(inPos++) & 255;
+            if (ctrl < MAX_LITERAL) {
+                // literal run of length = ctrl + 1,
+                ctrl++;
+                // copy to output and move forward this many bytes
+                System.arraycopy(in, inPos, out, outPos, ctrl);
+                outPos += ctrl;
+                inPos += ctrl;
+            } else {
+                // back reference
+                // the highest 3 bits are the match length
+                int len = ctrl >> 5;
+                // if the length is maxed, add the next byte to the length
+                if (len == 7) {
+                    len += in.get(inPos++) & 255;
+                }
+                // minimum back-reference is 3 bytes,
+                // so 2 was subtracted before storing size
+                len += 2;
+
+                // ctrl is now the offset for a back-reference...
+                // the logical AND operation removes the length bits
+                ctrl = -((ctrl & 0x1f) << 8) - 1;
+
+                // the next byte augments/increases the offset
+                ctrl -= in.get(inPos++) & 255;
+
+                // copy the back-reference bytes from the given
+                // location in output to current position
+                ctrl += outPos;
+                if (outPos + len >= out.capacity()) {
+                    // reduce array bounds checking
+                    throw new ArrayIndexOutOfBoundsException();
+                }
+                for (int i = 0; i < len; i++) {
+                    out.put(outPos++, out.get(ctrl++));
+                }
+            }
+        } while (outPos < outLen);
+    }
+    
     public int getAlgorithm() {
         return Compressor.LZF;
     }

h2/src/main/org/h2/store/fs/FilePath.java

@@ -72,6 +72,8 @@ public abstract class FilePath {
                     "org.h2.store.fs.FilePathDisk",
                     "org.h2.store.fs.FilePathMem",
                     "org.h2.store.fs.FilePathMemLZF",
+                    "org.h2.store.fs.FilePathNioMem",
+                    "org.h2.store.fs.FilePathNioMemLZF",
                     "org.h2.store.fs.FilePathSplit",
                     "org.h2.store.fs.FilePathNio",
                     "org.h2.store.fs.FilePathNioMapped",

h2/src/test/org/h2/test/TestBase.java

@@ -152,7 +152,9 @@ public abstract class TestBase {
         } finally {
             try {
                 FileUtils.deleteRecursive("memFS:", false);
+                FileUtils.deleteRecursive("nioMemFS:", false);
                 FileUtils.deleteRecursive("memLZF:", false);
+                FileUtils.deleteRecursive("nioMemLZF:", false);
             } catch (RuntimeException e) {
                 e.printStackTrace();
             }

h2/src/test/org/h2/test/unit/TestFile.java

@@ -31,17 +31,19 @@ public class TestFile extends TestBase implements DataHandler {
     }
 
     public void test() throws Exception {
-        doTest(false);
-        doTest(true);
+        doTest(false, false);
+        doTest(false, true);
+        doTest(true, false);
+        doTest(true, true);
     }
 
-    private void doTest(boolean compress) throws Exception {
+    private void doTest(boolean nioMem, boolean compress) throws Exception {
         int len = getSize(1000, 10000);
         Random random = new Random();
         FileStore mem = null, file = null;
         byte[] buffMem = null;
         byte[] buffFile = null;
-        String prefix = compress ? "memLZF:" : "memFS:";
+        String prefix = nioMem ? (compress ? "nioMemLZF:" : "nioMemFS:") : (compress ? "memLZF:" : "memFS:");
         FileUtils.delete(prefix + "test");
         FileUtils.delete("~/testFile");
 

h2/src/test/org/h2/test/unit/TestFileSystem.java

@@ -70,7 +70,9 @@ public class TestFileSystem extends TestBase {
         FileUtils.toRealPath(f);
         testFileSystem(getBaseDir() + "/fs");
         testFileSystem("memFS:");
+        testFileSystem("nioMemFS:");
         testFileSystem("memLZF:");
+        testFileSystem("nioMemLZF:");
         testUserHome();
         try {
             testFileSystem("nio:" + getBaseDir() + "/fs");
@@ -448,7 +450,8 @@ public class TestFileSystem extends TestBase {
 
         assertTrue(FileUtils.tryDelete(fsBase + "/test2"));
         FileUtils.delete(fsBase + "/test");
-        if (fsBase.indexOf("memFS:") < 0 && fsBase.indexOf("memLZF:") < 0) {
+        if (fsBase.indexOf("memFS:") < 0 && fsBase.indexOf("memLZF:") < 0
+            && fsBase.indexOf("nioMemFS:") < 0 && fsBase.indexOf("nioMemLZF:") < 0) {
             FileUtils.createDirectories(fsBase + "/testDir");
             assertTrue(FileUtils.isDirectory(fsBase + "/testDir"));
             if (!fsBase.startsWith("jdbc:")) {