File system abstraction: if used directly, some file systems did not work…

File system abstraction: if used directly, some file systems did not work correctly with spliced byte buffers (the database engine doesn't use those).

h2/src/main/org/h2/store/fs/FileChannelInputStream.java

@@ -18,7 +18,9 @@ public class FileChannelInputStream extends InputStream {
 
     private final FileChannel channel;
     private final boolean closeChannel;
-    private final byte[] buffer = { 0 };
+    
+    private ByteBuffer buffer;
+    private long pos;
 
     /**
      * Create a new file object input stream from the file channel.
@@ -33,11 +35,15 @@ public class FileChannelInputStream extends InputStream {
 
     @Override
     public int read() throws IOException {
-        if (channel.position() >= channel.size()) {
+        if (buffer == null) {
+            buffer = ByteBuffer.allocate(1);            
+        }
+        buffer.rewind();
+        int len = channel.read(buffer, pos++);
+        if (len < 0) {
             return -1;
         }
-        FileUtils.readFully(channel, ByteBuffer.wrap(buffer));
-        return buffer[0] & 0xff;
+        return buffer.get(0) & 0xff;
     }
 
     @Override
@@ -47,11 +53,13 @@ public class FileChannelInputStream extends InputStream {
 
     @Override
     public int read(byte[] b, int off, int len) throws IOException {
-        if (channel.position() + len < channel.size()) {
-            FileUtils.readFully(channel, ByteBuffer.wrap(b, off, len));
-            return len;
+        ByteBuffer buff = ByteBuffer.wrap(b, off, len);
+        int read = channel.read(buff, pos);
+        if (read == -1) {
+            return -1;
         }
-        return super.read(b, off, len);
+        pos += read;
+        return read;
     }
 
     @Override

h2/src/main/org/h2/store/fs/FilePathCrypt.java

@@ -256,7 +256,7 @@ public class FilePathCrypt extends FilePathWrapper {
             readFully(base, position + HEADER_LENGTH, dst);
             long block = position / BLOCK_SIZE;
             while (len > 0) {
-                xts.decrypt(block++, BLOCK_SIZE, dst.array(), x);
+                xts.decrypt(block++, BLOCK_SIZE, dst.array(), dst.arrayOffset() + x);
                 x += BLOCK_SIZE;
                 len -= BLOCK_SIZE;
             }
@@ -317,7 +317,7 @@ public class FilePathCrypt extends FilePathWrapper {
             long block = position / BLOCK_SIZE;
             int x = 0, l = len;
             while (l > 0) {
-                xts.encrypt(block++, BLOCK_SIZE, crypt.array(), x);
+                xts.encrypt(block++, BLOCK_SIZE, crypt.array(), crypt.arrayOffset() + x);
                 x += BLOCK_SIZE;
                 l -= BLOCK_SIZE;
             }

h2/src/main/org/h2/store/fs/FilePathDisk.java

@@ -443,7 +443,7 @@ class FileDisk extends FileBase {
 
     @Override
     public int read(ByteBuffer dst) throws IOException {
-        int len = file.read(dst.array(), dst.position(), dst.remaining());
+        int len = file.read(dst.array(), dst.arrayOffset() + dst.position(), dst.remaining());
         if (len > 0) {
             dst.position(dst.position() + len);
         }
@@ -459,7 +459,7 @@ class FileDisk extends FileBase {
     @Override
     public int write(ByteBuffer src) throws IOException {
         int len = src.remaining();
-        file.write(src.array(), src.position(), len);
+        file.write(src.array(), src.arrayOffset() + src.position(), len);
         src.position(src.position() + len);
         return len;
     }

h2/src/main/org/h2/store/fs/FilePathMem.java

@@ -274,7 +274,7 @@ class FileMem extends FileBase {
             return 0;
         }
         data.touch(readOnly);
-        pos = data.readWrite(pos, src.array(), src.position(), len, true);
+        pos = data.readWrite(pos, src.array(), src.arrayOffset() + src.position(), len, true);
         src.position(src.position() + len);
         return len;
     }
@@ -285,7 +285,7 @@ class FileMem extends FileBase {
         if (len == 0) {
             return 0;
         }
-        long newPos = data.readWrite(pos, dst.array(), dst.position(), len, false);
+        long newPos = data.readWrite(pos, dst.array(), dst.arrayOffset() + dst.position(), len, false);
         len = (int) (newPos - pos);
         if (len <= 0) {
             return -1;

h2/src/main/org/h2/store/fs/FilePathNioMapped.java

@@ -175,7 +175,7 @@ class FileNioMapped extends FileBase {
                 return -1;
             }
             mapped.position(pos);
-            mapped.get(dst.array(), dst.position(), len);
+            mapped.get(dst.array(), dst.arrayOffset() + dst.position(), len);
             dst.position(dst.position() + len);
             pos += len;
             return len;

h2/src/main/org/h2/store/fs/FilePathRec.java

@@ -178,7 +178,7 @@ class FileRec extends FileBase {
         int len = src.remaining();
         if (src.position() != 0 || len != buff.length) {
             byte[] b = new byte[len];
-            System.arraycopy(buff, src.position(), b, 0, len);
+            System.arraycopy(buff, src.arrayOffset() + src.position(), b, 0, len);
             buff = b;
         }
         int result = channel.write(src);

h2/src/main/org/h2/store/fs/FilePathZip.java

@@ -281,7 +281,7 @@ class FileZip extends FileBase {
     @Override
     public int read(ByteBuffer dst) throws IOException {
         seek();
-        int len = in.read(dst.array(), dst.position(), dst.remaining());
+        int len = in.read(dst.array(), dst.arrayOffset() + dst.position(), dst.remaining());
         if (len > 0) {
             dst.position(dst.position() + len);
             pos += len;

h2/src/test/org/h2/test/unit/TestFileSystem.java

@@ -22,7 +22,13 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.List;
 import java.util.Random;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.h2.dev.fs.FilePathZip2;
 import org.h2.message.DbException;
+import org.h2.mvstore.DataUtils;
+import org.h2.mvstore.cache.FilePathCache;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FilePathCrypt;
 import org.h2.store.fs.FileUtils;
@@ -57,11 +63,16 @@ public class TestFileSystem extends TestBase {
         testDirectories(getBaseDir());
         testMoveTo(getBaseDir());
         testUnsupportedFeatures(getBaseDir());
+        FilePathZip2.register();
+        FilePath.register(new FilePathCache());
+        testZipFileSystem("zip:");
+        testZipFileSystem("cache:zip:");
+        testZipFileSystem("zip2:");
+        testZipFileSystem("cache:zip2:");
         testMemFsDir();
         testClasspath();
         FilePathDebug.register().setTrace(true);
         FilePathCrypt.register();
-        testFileSystem("crypt:0007:" + getBaseDir() + "/fs");
         testSimpleExpandTruncateSize();
         testSplitDatabaseInZip();
         testDatabaseInMemFileSys();
@@ -77,7 +88,10 @@ public class TestFileSystem extends TestBase {
         testUserHome();
         try {
             testFileSystem("nio:" + getBaseDir() + "/fs");
+            testFileSystem("cache:nio:" + getBaseDir() + "/fs");
             testFileSystem("nioMapped:" + getBaseDir() + "/fs");
+            testFileSystem("crypt:0007:" + getBaseDir() + "/fs");
+            testFileSystem("cache:crypt:0007:" + getBaseDir() + "/fs");
             if (!config.splitFileSystem) {
                 testFileSystem("split:" + getBaseDir() + "/fs");
                 testFileSystem("split:nioMapped:" + getBaseDir() + "/fs");
@@ -93,6 +107,98 @@ public class TestFileSystem extends TestBase {
         }
     }
     
+    private void testZipFileSystem(String prefix) throws IOException {
+        Random r = new Random(1);
+        for (int i = 0; i < 5; i++) {
+            testZipFileSystem(prefix, r);
+        }
+    }
+    
+    private void testZipFileSystem(String prefix, Random r) throws IOException {
+        byte[] data = new byte[r.nextInt(16 * 1024)];
+        long x = r.nextLong();
+        FilePath file = FilePath.get(getBaseDir() + "/fs/readonly" + x + ".zip");
+        r.nextBytes(data);
+        OutputStream out = file.newOutputStream(false);
+        ZipOutputStream zipOut = new ZipOutputStream(out);
+        ZipEntry entry = new ZipEntry("data");
+        zipOut.putNextEntry(entry);
+        zipOut.write(data);
+        zipOut.closeEntry();
+        zipOut.close();
+        out.close();
+        FilePath fp = FilePath.get(
+                prefix + getBaseDir() + "/fs/readonly" + x + ".zip!data");
+        FileChannel fc = fp.open("r");
+        StringBuilder buff = new StringBuilder();
+        try {
+            int pos = 0;
+            for (int i = 0; i < 100; i++) {
+                trace("op " + i);
+                switch (r.nextInt(5)) {
+                case 0: {
+                    int p = r.nextInt(data.length);
+                    trace("seek " + p);
+                    buff.append("seek " + p + "\n");
+                    fc.position(p);
+                    pos = p;
+                    break;
+                }
+                case 1: {
+                    int len = r.nextInt(1000);
+                    int offset = r.nextInt(100);
+                    int arrayLen = len + offset;
+                    len = Math.min(len, data.length - pos);
+                    byte[] b1 = new byte[arrayLen];
+                    byte[] b2 = new byte[arrayLen];
+                    trace("readFully " + len);
+                    buff.append("readFully " + len + "\n");
+                    System.arraycopy(data, pos, b1, offset, len);
+                    ByteBuffer byteBuff = createSlicedBuffer(b2, offset, len);
+                    FileUtils.readFully(fc, byteBuff);
+                    assertEquals(b1, b2);
+                    pos += len;
+                    break;
+                }
+                case 2: {
+                    int len = r.nextInt(1000);
+                    int offset = r.nextInt(100);
+                    int arrayLen = len + offset;
+                    int p = r.nextInt(data.length);
+                    len = Math.min(len, data.length - p);
+                    byte[] b1 = new byte[arrayLen];
+                    byte[] b2 = new byte[arrayLen];
+                    trace("readFully " + p + " " + len);
+                    buff.append("readFully " + p + " " + len + "\n");
+                    System.arraycopy(data, p, b1, offset, len);
+                    ByteBuffer byteBuff = createSlicedBuffer(b2, offset, len);
+                    DataUtils.readFully(fc, p, byteBuff);
+                    assertEquals(b1, b2);
+                    break;
+                }
+                case 3: {
+                    trace("getFilePointer");
+                    buff.append("getFilePointer\n");
+                    assertEquals(pos, fc.position());
+                    break;
+                }
+                case 4: {
+                    trace("length " + data.length);
+                    buff.append("length " + data.length + "\n");
+                    assertEquals(data.length, fc.size());
+                    break;
+                }
+                default:
+                }
+            }
+            fc.close();
+            file.delete();
+        } catch (Throwable e) {
+            e.printStackTrace();
+            fail("Exception: " + e + "\n"+ buff.toString());
+        }
+    }
+
     private void testAbsoluteRelative() {
         assertFalse(FileUtils.isAbsolute("test/abc"));
         assertTrue(FileUtils.isAbsolute("~/test/abc"));
@@ -512,12 +618,16 @@ public class TestFileSystem extends TestBase {
                     break;
                 }
                 case 1: {
-                    byte[] buffer = new byte[random.nextInt(1000)];
+                    int arrayLen = random.nextInt(1000);
+                    int offset = random.nextInt(arrayLen / 10);
+                    int len = random.nextInt(arrayLen - offset);
+                    byte[] buffer = new byte[arrayLen];
+                    ByteBuffer byteBuff = createSlicedBuffer(buffer, offset, len);
                     random.nextBytes(buffer);
-                    trace("write " + buffer.length);
-                    buff.append("write " + buffer.length + "\n");
-                    f.write(ByteBuffer.wrap(buffer));
-                    ra.write(buffer, 0, buffer.length);
+                    trace("write " + offset + " len " + len);
+                    buff.append("write " + offset + " " + len + "\n");
+                    f.write(byteBuff);
+                    ra.write(buffer, offset, len);
                     break;
                 }
                 case 2: {
@@ -534,13 +644,16 @@ public class TestFileSystem extends TestBase {
                 }
                 case 3: {
                     int len = random.nextInt(1000);
+                    int offset = random.nextInt(100);
+                    int arrayLen = len + offset;
                     len = (int) Math.min(len, ra.length() - ra.getFilePointer());
-                    byte[] b1 = new byte[len];
-                    byte[] b2 = new byte[len];
+                    byte[] b1 = new byte[arrayLen];
+                    byte[] b2 = new byte[arrayLen];
                     trace("readFully " + len);
                     buff.append("readFully " + len + "\n");
-                    ra.readFully(b1, 0, len);
-                    FileUtils.readFully(f, ByteBuffer.wrap(b2, 0, len));
+                    ra.readFully(b1, offset, len);
+                    ByteBuffer byteBuff = createSlicedBuffer(b2, offset, len);
+                    FileUtils.readFully(f, byteBuff);
                     assertEquals(b1, b2);
                     break;
                 }
@@ -580,6 +693,15 @@ public class TestFileSystem extends TestBase {
         }
     }
     
+    private static ByteBuffer createSlicedBuffer(byte[] buffer, int offset, int len) {
+        ByteBuffer byteBuff = ByteBuffer.wrap(buffer);
+        byteBuff.position(offset);
+        // force the arrayOffset to be non-0
+        byteBuff = byteBuff.slice();
+        byteBuff.limit(len);        
+        return byteBuff;
+    }
+
     private void testTempFile(String fsBase) throws Exception {
         int len = 10000;
         String s = FileUtils.createTempFile(fsBase + "/tmp", ".tmp", false, false);

h2/src/tools/org/h2/dev/fs/FilePathZip2.java

@@ -29,8 +29,7 @@ import org.h2.util.New;
 /**
  * This is a read-only file system that allows to access databases stored in a
  * .zip or .jar file. The problem of this file system is that data is always
- * accessed as a stream. But unlike FileSystemZip, it is possible to stack file
- * systems.
+ * accessed as a stream. This implementation allows to stack file systems.
  */
 public class FilePathZip2 extends FilePath {
 
@@ -363,7 +362,7 @@ class FileZip2 extends FileBase {
     @Override
     public int read(ByteBuffer dst) throws IOException {
         seek();
-        int len = in.read(dst.array(), dst.position(), dst.remaining());
+        int len = in.read(dst.array(), dst.arrayOffset() + dst.position(), dst.remaining());
         if (len > 0) {
             dst.position(dst.position() + len);
             pos += len;
@@ -447,7 +446,7 @@ class FileZip2 extends FileBase {
 
     @Override
     public String toString() {
-        return name;
+        return "zip2:" + name;
     }
 
 }
\ No newline at end of file