CompressLZF: optimized expand method and improved javadocs

h2/src/docsrc/html/changelog.html

@@ -30,6 +30,7 @@ Change Log
     A large binary is abbreviated as follows: abcdef... (100000 bytes).
 </li><li>Faster data conversion from BIGINT or INT to DECIMAL.
 </li><li>Server-less multi-connection mode: try to delete log files if switching between read/write operations.
+</li><li>CompressLZF: Faster decompress and improved javadocs
 </li></ul>
 
 <h2>Version 1.2.126 (2009-12-18)</h2>

h2/src/main/org/h2/compress/CompressLZF.java

@@ -33,39 +33,99 @@
 
 package org.h2.compress;
 
+import java.sql.SQLException;
+
 /**
  * This class implements the LZF lossless data compression algorithm.
- * LZF is optimized for speed.
+ * LZF is a Lempel-Ziv variant with byte-aligned output, and optimized for speed.
+ *
+ * <h2>Safety/Use Notes:</h2>
+ * <ul><li> Each instance should be used by a single thread only,
+ *   due to cached hashtable</li>
+ *     <li> May run into problems when data buffers approach Integer.MAX_VALUE
+ *       (or, say, 2^31)</li>
+ *     <li> For performance reasons, safety checks on expansion omitted</li>
+ *     <li> Invalid compressed data can cause ArrayIndexOutOfBoundsException</li>
+ * </ul>
+ * <p />
+ * <h2>LZF compressed format:</h2>
+ * <ul><li>2 modes: literal run, or back-reference to previous data
+ *     <ul><li>Literal run: directly copy bytes from input to output</li>
+ *         <li>Back-reference: copy previous data to output stream,
+ *           with specified offset from location and length</li>
+ *     </ul>
+ * </li>
+ * <li>Back-references are assumed to be at least 3 bytes,
+ *   otherwise there is no gain from using a back-reference.</li>
+ * </ul>
+ * <h2>Binary format:</h2>
+ * <ul><li>First byte -- control byte:
+ *    <ul><li>highest 3 bits are back-reference length, or 0 if literal run</li>
+ *     <li>lowest 5 bits are either literal run length or
+ *       part of offset for back-reference</li>
+ *    </ul></li>
+ * <li>If literal run:
+ *    <ul><li> next bytes are data to copy directly into output</li></ul>
+ * </li>
+ * <li>If back reference:
+ *    <ul><li>If and only if back reference length is 7 (top 3 bits set),
+ *      add next byte to back reference length as unsigned byte</li>
+ *     <li>In either case, add next byte to offset location
+ *       with lowest 5 bits of control byte</li>
+ *    </ul></li>
+ * </ul>
  */
-public class CompressLZF implements Compressor {
+public final class CompressLZF implements Compressor {
 
+    /** Number of entries for main hash table
+     * <br />Size is a trade-off between hash collisions (reduced compression)
+     * and speed (amount that fits in CPU cache) */
     private static final int HASH_SIZE = 1 << 14;
+
+    /** 32: maximum number of literals in a chunk */
     private static final int MAX_LITERAL = 1 << 5;
+
+    /** 8192, maximum offset allowed for a back-reference */
     private static final int MAX_OFF = 1 << 13;
+
+    /** Maximum back-reference length
+     * == 256 (full byte) + 8 (top 3 bits of byte) + 1 = 264 */
     private static final int MAX_REF = (1 << 8) + (1 << 3);
 
+    /** Hash table for matching byte sequences -- reused for performance */
     private int[] cachedHashTable;
 
-    public void setOptions(String options) {
+    public void setOptions(String options) throws SQLException {
         // nothing to do
     }
 
-    public int getAlgorithm() {
-        return Compressor.LZF;
+    /**
+     * Return byte with lower 2 bytes being byte at index, then index+1
+     */
+    private static int first(byte[] in, int inPos) {
+        return (in[inPos] << 8) | (in[inPos + 1] & 255);
     }
 
-    private int first(byte[] in, int inPos) {
-        return (in[inPos] << 8) + (in[inPos + 1] & 255);
+    /**
+     * Shift v 1 byte left, add value at index inPos+2
+     */
+    private static int next(int v, byte[] in, int inPos) {
+        return (v << 8) | (in[inPos + 2] & 255);
     }
 
-    private int next(int v, byte[] in, int inPos) {
-        return (v << 8) + (in[inPos + 2] & 255);
-    }
-
-    private int hash(int h) {
+    /** Compute address in hash table */
+    private static int hash(int h) {
         return ((h * 2777) >> 9) & (HASH_SIZE - 1);
     }
 
+    /**
+     * Compress from one buffer to another
+     * @param in Input buffer
+     * @param inLen Length of bytes to compress from input buffer
+     * @param out Output buffer
+     * @param outPos Starting position in out buffer
+     * @return Number of bytes written to output buffer
+     */
     public int compress(byte[] in, int inLen, byte[] out, int outPos) {
         int inPos = 0;
         if (cachedHashTable == null) {
@@ -74,28 +134,31 @@ public class CompressLZF implements Compressor {
         int[] hashTab = cachedHashTable;
         int literals = 0;
         outPos++;
-        int hash = first(in, 0);
+        int future = first(in, 0);
         while (inPos < inLen - 4) {
             byte p2 = in[inPos + 2];
             // next
-            hash = (hash << 8) + (p2 & 255);
-            int off = hash(hash);
+            future = (future << 8) + (p2 & 255);
+            int off = hash(future);
             int ref = hashTab[off];
             hashTab[off] = inPos;
             if (ref < inPos
                         && ref > 0
                         && (off = inPos - ref - 1) < MAX_OFF
                         && in[ref + 2] == p2
-                        && in[ref + 1] == (byte) (hash >> 8)
-                        && in[ref] == (byte) (hash >> 16)) {
+                        && in[ref + 1] == (byte) (future >> 8)
+                        && in[ref] == (byte) (future >> 16)) {
                 // match
                 int maxLen = inLen - inPos - 2;
                 if (maxLen > MAX_REF) {
                     maxLen = MAX_REF;
                 }
                 if (literals == 0) {
+                    // back-to-back back-reference, so no control byte for literal run
                     outPos--;
                 } else {
+                    // set control byte at start of literal run
+                    // to store number of literals
                     out[outPos - literals - 1] = (byte) (literals - 1);
                     literals = 0;
                 }
@@ -111,23 +174,31 @@ public class CompressLZF implements Compressor {
                     out[outPos++] = (byte) (len - 7);
                 }
                 out[outPos++] = (byte) off;
+                // move one byte forward to allow for control byte on next literal run
                 outPos++;
                 inPos += len;
-                hash = first(in, inPos);
-                hash = next(hash, in, inPos);
-                hashTab[hash(hash)] = inPos++;
-                hash = next(hash, in, inPos);
-                hashTab[hash(hash)] = inPos++;
+                // rebuild the future, and store last couple bytes to hashtable
+                // storing hashes of last bytes in back-reference improves compression ratio
+                // and only reduces speed *slightly*
+                future = first(in, inPos);
+                future = next(future, in, inPos);
+                hashTab[hash(future)] = inPos++;
+                future = next(future, in, inPos);
+                hashTab[hash(future)] = inPos++;
             } else {
+                // copy byte from input to output as part of literal
                 out[outPos++] = in[inPos++];
                 literals++;
+                // end of this literal chunk, write length to control byte and start new chunk
                 if (literals == MAX_LITERAL) {
                     out[outPos - literals - 1] = (byte) (literals - 1);
                     literals = 0;
+                    // move ahead one byte to allow for control byte containing literal length
                     outPos++;
                 }
             }
         }
+        // writes out remaining few bytes as literals
         while (inPos < inLen) {
             out[outPos++] = in[inPos++];
             literals++;
@@ -137,6 +208,7 @@ public class CompressLZF implements Compressor {
                 outPos++;
             }
         }
+        // writes final literal run length to control byte
         out[outPos - literals - 1] = (byte) (literals - 1);
         if (literals == 0) {
             outPos--;
@@ -144,40 +216,53 @@ public class CompressLZF implements Compressor {
         return outPos;
     }
 
+    /**
+     * Expand compressed data from one buffer to another
+     * @param in Compressed data buffer
+     * @param inPos Index of first byte in input data
+     * @param inLen Number of compressed input bytes
+     * @param out Output buffer for decompressed data
+     * @param outPos Index for start of decompressed data
+     * @param outLen Size of decompressed data
+     */
     public void expand(byte[] in, int inPos, int inLen, byte[] out, int outPos, int outLen) {
         do {
             int ctrl = in[inPos++] & 255;
-            if (ctrl < (1 << 5)) {
-                // literal run
-                ctrl += inPos;
-                do {
-                    out[outPos++] = in[inPos];
-                } while (inPos++ < ctrl);
+            // literal run of length = ctrl + 1,
+            //  directly copy to output and move forward this many bytes
+            if (ctrl < MAX_LITERAL) {
+                ctrl++;
+                System.arraycopy(in, inPos, out, outPos, ctrl);
+                outPos += ctrl;
+                inPos += ctrl;
             } else {
                 // back reference
+                // highest 3 bits are match length
                 int len = ctrl >> 5;
-                ctrl = -((ctrl & 0x1f) << 8) - 1;
+                // if length is maxed add in next byte to length
                 if (len == 7) {
                     len += in[inPos++] & 255;
                 }
+                // minimum back-reference is 3 bytes, so 2 was subtracted before storing size
+                len += 2;
+
+                // control is now offset amount for back-reference...
+                // the logical AND operation removes the length bits
+                ctrl = -((ctrl & 0x1f) << 8) - 1;
+
+                // next byte augments/increases offset
                 ctrl -= in[inPos++] & 255;
-                len += outPos + 2;
-                out[outPos] = out[outPos++ + ctrl];
-                out[outPos] = out[outPos++ + ctrl];
-                while (outPos < len - 8) {
-                    out[outPos] = out[outPos++ + ctrl];
-                    out[outPos] = out[outPos++ + ctrl];
-                    out[outPos] = out[outPos++ + ctrl];
-                    out[outPos] = out[outPos++ + ctrl];
-                    out[outPos] = out[outPos++ + ctrl];
-                    out[outPos] = out[outPos++ + ctrl];
-                    out[outPos] = out[outPos++ + ctrl];
-                    out[outPos] = out[outPos++ + ctrl];
-                }
-                while (outPos < len) {
-                    out[outPos] = out[outPos++ + ctrl];
+
+                // quickly copy back-reference bytes from location in output to current position
+                for (int i = 0; i < len; i++) {
+                    out[outPos + i] = out[outPos + ctrl + i];
                 }
+                outPos += len;
             }
         } while (outPos < outLen);
     }
-}
+
+    public int getAlgorithm() {
+        return Compressor.LZF;
+    }
+}
\ No newline at end of file

h2/src/tools/org/h2/build/doc/dictionary.txt

@@ -625,3 +625,5 @@ weakreference ancestor junctions wake fills rail sleeps turns grammars straight
 answers attachments emails clipboard prioritize tips urgently standby
 checklists serves gbif biodiversity wakes taxon ratio ended ipt auckland
 galapagos pacific pastebin mystic posting mysticpaste reject prof tick
+hashes decompressed expansion ziv abbreviated augments omitted gain
+subtracted maxed logical lempel increases
\ No newline at end of file