File systems with a maximum file size (for example FAT) are now supported.

h2/src/main/org/h2/store/fs/FileObject.java

@@ -68,4 +68,10 @@ public interface FileObject {
      */
     void setFileLength(long newLength) throws IOException;
 
+    /**
+     * Get the full qualified name of this file.
+     *
+     * @return the name
+     */
+    String getName();
 }

h2/src/main/org/h2/store/fs/FileObjectDatabase.java

@@ -85,4 +85,8 @@ public class FileObjectDatabase implements FileObject {
         changed = true;
     }
     
+    public String getName() {
+        return fileName;
+    }
+
 }

h2/src/main/org/h2/store/fs/FileObjectDisk.java

@@ -17,8 +17,11 @@ import org.h2.util.FileUtils;
  */
 public class FileObjectDisk extends RandomAccessFile implements FileObject {
     
-    public FileObjectDisk(String fileName, String mode) throws FileNotFoundException {
+    private final String name;
+
+    FileObjectDisk(String fileName, String mode) throws FileNotFoundException {
         super(fileName, mode);
+        this.name = fileName;
     }
 
     public void sync() throws IOException {
@@ -29,4 +32,8 @@ public class FileObjectDisk extends RandomAccessFile implements FileObject {
         FileUtils.setLength(this, newLength);
     }
 
+    public String getName() {
+        return name;
+    }
+
 }

h2/src/main/org/h2/store/fs/FileObjectMemory.java

@@ -89,7 +89,7 @@ public class FileObjectMemory implements FileObject {
     }
 //## Java 1.4 end ##
     
-    public FileObjectMemory(String name, boolean compress) {
+    FileObjectMemory(String name, boolean compress) {
         this.name = name;
         this.compress = compress;
         data = new byte[0][];

h2/src/main/org/h2/store/fs/FileObjectSplit.java

+/*
+ * Copyright 2004-2008 H2 Group. Multiple-Licensed under the H2 License, 
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.store.fs;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import org.h2.message.Message;
+import org.h2.util.FileUtils;
+
+/**
+ * A file that may be split into multiple smaller files.
+ */
+public class FileObjectSplit implements FileObject {
+    
+    private final String name;
+    private final String mode;
+    private final long maxLength;
+    private FileObject[] list;
+    private long filePointer;
+    private long length;
+    
+    FileObjectSplit(String name, String mode, FileObject[] list, long length, long maxLength) {
+        this.name = name;
+        this.mode = mode;
+        this.list = list;
+        this.length = length;
+        this.maxLength = maxLength;
+    }
+    
+    public void close() throws IOException {
+        for (int i = 0; i < list.length; i++) {
+            list[i].close();
+        }
+    }
+
+    public long getFilePointer() throws IOException {
+        return filePointer;
+    }
+
+    public long length() throws IOException {
+        return length;
+    }
+    
+    private int read(byte[] b, int off, int len) throws IOException {
+        long offset = filePointer % maxLength;
+        int l = (int) Math.min(len, maxLength - offset);
+        FileObject fo = getFileObject();
+        fo.seek(offset);
+        fo.readFully(b, off, l);
+        filePointer += l;
+        return l;
+    }
+
+    public void readFully(byte[] b, int off, int len) throws IOException {
+        while (true) {
+            int l = read(b, off, len);
+            len -= l;
+            if (len <= 0) {
+                return;
+            }
+            off += l;
+        }
+    }
+
+    public void seek(long pos) throws IOException {
+        filePointer = pos;
+    }
+    
+    private FileObject getFileObject() throws IOException {
+        int id = (int) (filePointer / maxLength);
+        while (id >= list.length) {
+            int i = list.length;
+            FileObject[] newList = new FileObject[i + 1];
+            System.arraycopy(list, 0, newList, 0, i);
+            String fileName = FileSystemSplit.getFileName(name, i);
+            newList[i] = FileSystem.getInstance(fileName).openFileObject(fileName, mode);
+            list = newList;
+        }
+        return list[id];
+    }
+
+    public void setFileLength(long newLength) throws IOException {
+        filePointer = Math.min(filePointer, newLength);
+        int newFileCount = 1 + (int) (newLength / maxLength);
+        if (newFileCount == list.length) {
+            long size = newLength - maxLength * (newFileCount - 1);
+            list[list.length - 1].setFileLength(size);
+        } else {
+            FileObject[] newList = new FileObject[newFileCount];
+            int max = Math.max(newFileCount, list.length);
+            long remaining = newLength;
+            for (int i = 0; i < max; i++) {
+                long size = Math.min(remaining, maxLength);
+                remaining -= size;
+                if (i >= newFileCount) {
+                    list[i].close();
+                    try {
+                        FileUtils.delete(list[i].getName());
+                    } catch (SQLException e) {
+                        throw Message.convertToIOException(e);
+                    }
+                } else if (i >= list.length) {
+                    String fileName = FileSystemSplit.getFileName(name, i);
+                    FileObject o = FileSystem.getInstance(fileName).openFileObject(fileName, mode);
+                    o.setFileLength(size);
+                    newList[i] = o;
+                } else {
+                    FileObject o = list[i];
+                    if (o.length() != size) {
+                        o.setFileLength(size);
+                    }
+                    newList[i] = list[i];
+                }
+            }
+            list = newList;
+        }
+        this.length = newLength;
+    }
+
+    public void sync() throws IOException {
+        for (int i = 0; i < list.length; i++) {
+            list[i].sync();
+        }
+    }
+
+    public void write(byte[] b, int off, int len) throws IOException {
+        while (true) {
+            int l = writePart(b, off, len);
+            len -= l;
+            if (len <= 0) {
+                return;
+            }
+            off += l;
+        }
+    }
+    
+    private int writePart(byte[] b, int off, int len) throws IOException {
+        long offset = filePointer % maxLength;
+        int l = (int) Math.min(len, maxLength - offset);
+        FileObject fo = getFileObject();
+        fo.seek(offset);
+        fo.write(b, off, l);
+        filePointer += l;
+        length = Math.max(length, filePointer);
+        return l;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+}

h2/src/main/org/h2/store/fs/FileObjectZip.java

@@ -28,7 +28,7 @@ public class FileObjectZip implements FileObject {
     private long inPos;
     private long length;
 
-    public FileObjectZip(ZipFile file, ZipEntry entry) {
+    FileObjectZip(ZipFile file, ZipEntry entry) {
         this.file = file;
         this.entry = entry;
         length = entry.getSize();
@@ -85,4 +85,8 @@ public class FileObjectZip implements FileObject {
         throw new IOException("File is read-only");
     }
 
+    public String getName() {
+        return file.getName();
+    }
+
 }

h2/src/main/org/h2/store/fs/FileSystem.java

@@ -36,6 +36,12 @@ public abstract class FileSystem {
      */
     public static final String PREFIX_ZIP = "zip:";
 
+    /**
+     * The prefix used to split large files (required for a FAT32 because it
+     * only support files up to 2 GB).
+     */
+    public static final String PREFIX_SPLIT = "split:";
+
     /**
      * Get the file system object.
      *
@@ -49,6 +55,8 @@ public abstract class FileSystem {
             return FileSystemDatabase.getInstance(fileName);
         } else if (fileName.startsWith(PREFIX_ZIP)) {
             return FileSystemZip.getInstance();
+        } else if (fileName.startsWith(PREFIX_SPLIT)) {
+            return FileSystemSplit.getInstance();
         }
         return FileSystemDisk.getInstance();
     }

h2/src/main/org/h2/store/fs/FileSystemSplit.java

+/*
+ * Copyright 2004-2008 H2 Group. Multiple-Licensed under the H2 License, 
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.store.fs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.SequenceInputStream;
+import java.sql.SQLException;
+import java.util.ArrayList;
+
+import org.h2.constant.SysProperties;
+import org.h2.message.Message;
+
+/**
+ * A file system that may split files into multiple smaller files.
+ */
+public class FileSystemSplit extends FileSystem {
+
+    static final String PART_SUFFIX = ".part";
+    private static final FileSystemSplit INSTANCE = new FileSystemSplit();
+    
+    long defaultMaxSize = 1L << SysProperties.SPLIT_FILE_SIZE_SHIFT;
+
+    public static FileSystemSplit getInstance() {
+        return INSTANCE;
+    }
+    
+    public boolean canWrite(String fileName) {
+        fileName = translateFileName(fileName);
+        return getFileSystem(fileName).canWrite(fileName);
+    }
+
+    public void copy(String original, String copy) throws SQLException {
+        original = translateFileName(original);
+        copy = translateFileName(copy);
+        getFileSystem(original).copy(original, copy);
+        for (int i = 1;; i++) {
+            String o = getFileName(original, i);
+            if (getFileSystem(o).exists(o)) {
+                String c = getFileName(copy, i);
+                getFileSystem(o).copy(o, c);
+            } else {
+                break;
+            }
+        }
+    }
+
+    public void createDirs(String fileName) throws SQLException {
+        fileName = translateFileName(fileName);
+        getFileSystem(fileName).createDirs(fileName);
+    }
+
+    public boolean createNewFile(String fileName) throws SQLException {
+        fileName = translateFileName(fileName);
+        return getFileSystem(fileName).createNewFile(fileName);
+    }
+
+    public String createTempFile(String prefix, String suffix, boolean deleteOnExit, boolean inTempDir)
+            throws IOException {
+        prefix = translateFileName(prefix);
+        return FileSystem.PREFIX_SPLIT + getFileSystem(prefix).createTempFile(prefix, suffix, deleteOnExit, inTempDir);
+    }
+
+    public void delete(String fileName) throws SQLException {
+        fileName = translateFileName(fileName);
+        for (int i = 0;; i++) {
+            String f = getFileName(fileName, i);
+            if (getFileSystem(fileName).exists(f)) {
+                getFileSystem(fileName).delete(f);
+            } else {
+                break;
+            }
+        }
+    }
+
+    public void deleteRecursive(String directory) throws SQLException {
+        directory = translateFileName(directory);
+        getFileSystem(directory).deleteRecursive(directory);
+    }
+
+    public boolean exists(String fileName) {
+        fileName = translateFileName(fileName);
+        return getFileSystem(fileName).exists(fileName);
+    }
+
+    public boolean fileStartsWith(String fileName, String prefix) {
+        fileName = translateFileName(fileName);
+        prefix = translateFileName(prefix);
+        return getFileSystem(fileName).fileStartsWith(fileName, prefix);
+    }
+
+    public String getAbsolutePath(String fileName) {
+        fileName = translateFileName(fileName);
+        return FileSystem.PREFIX_SPLIT + getFileSystem(fileName).getAbsolutePath(fileName);
+    }
+
+    public String getFileName(String name) throws SQLException {
+        name = translateFileName(name);
+        return getFileSystem(name).getFileName(name);
+    }
+
+    public long getLastModified(String fileName) {
+        fileName = translateFileName(fileName);
+        long lastModified = 0;
+        for (int i = 0;; i++) {
+            String f = getFileName(fileName, i);
+            if (getFileSystem(fileName).exists(f)) {
+                long l = getFileSystem(fileName).getLastModified(fileName);
+                lastModified = Math.max(lastModified, l);
+            } else {
+                break;
+            }
+        }
+        return lastModified;
+    }
+
+    public String getParent(String fileName) {
+        fileName = translateFileName(fileName);
+        return FileSystem.PREFIX_SPLIT + getFileSystem(fileName).getParent(fileName);
+    }
+
+    public boolean isAbsolute(String fileName) {
+        fileName = translateFileName(fileName);
+        return getFileSystem(fileName).isAbsolute(fileName);
+    }
+
+    public boolean isDirectory(String fileName) {
+        fileName = translateFileName(fileName);
+        return getFileSystem(fileName).isDirectory(fileName);
+    }
+
+    public boolean isReadOnly(String fileName) {
+        fileName = translateFileName(fileName);
+        return getFileSystem(fileName).isReadOnly(fileName);
+    }
+
+    public long length(String fileName) {
+        fileName = translateFileName(fileName);
+        long length = 0;
+        for (int i = 0;; i++) {
+            String f = getFileName(fileName, i);
+            if (getFileSystem(fileName).exists(f)) {
+                length += getFileSystem(fileName).length(f);
+            } else {
+                break;
+            }
+        }
+        return length;
+    }
+
+    public String[] listFiles(String directory) throws SQLException {
+        directory = translateFileName(directory);
+        String[] array = getFileSystem(directory).listFiles(directory);
+        ArrayList list = new ArrayList();
+        for (int i = 0; i < array.length; i++) {
+            String f = array[i];
+            if (f.endsWith(PART_SUFFIX)) {
+                continue;
+            }
+            array[i] = f = FileSystem.PREFIX_SPLIT + f;
+            list.add(f);
+        }
+        if (list.size() != array.length) {
+            array = new String[list.size()];
+            list.toArray(array);
+        }
+        return array;
+    }
+
+    public String normalize(String fileName) throws SQLException {
+        fileName = translateFileName(fileName);
+        return FileSystem.PREFIX_SPLIT + getFileSystem(fileName).normalize(fileName);
+    }
+
+    public InputStream openFileInputStream(String fileName) throws IOException {
+        fileName = translateFileName(fileName);
+        InputStream input = getFileSystem(fileName).openFileInputStream(fileName);
+        for (int i = 1;; i++) {
+            String f = getFileName(fileName, i);
+            if (getFileSystem(f).exists(f)) {
+                InputStream i2 = getFileSystem(f).openFileInputStream(f);
+                input = new SequenceInputStream(input, i2);
+            } else {
+                break;
+            }
+        }
+        return input;
+    }
+    
+    public FileObject openFileObject(String fileName, String mode) throws IOException {
+        fileName = translateFileName(fileName);
+        ArrayList list = new ArrayList();
+        FileObject o = getFileSystem(fileName).openFileObject(fileName, mode);
+        list.add(o);
+        for (int i = 1;; i++) {
+            String f = getFileName(fileName, i);
+            if (getFileSystem(fileName).exists(f)) {
+                o = getFileSystem(f).openFileObject(f, mode);
+                list.add(o);
+            } else {
+                break;
+            }
+        }
+        FileObject[] array = new FileObject[list.size()];
+        list.toArray(array);
+        long maxLength = array[0].length();
+        long length = maxLength;
+        if (array.length == 1) {
+            if (maxLength < defaultMaxSize) {
+                maxLength = defaultMaxSize;
+            }
+        } else {
+            for (int i = 1; i < array.length - 1; i++) {
+                o = array[i];
+                long l = o.length();
+                length += l;
+                if (l != maxLength) {
+                    throw new IOException("Expected file length: " + maxLength + " got: " + l + " for " + o.getName());
+                }
+            }
+            o = array[array.length - 1];
+            long l = o.length();
+            length += l;
+            if (l > maxLength) {
+                throw new IOException("Expected file length: " + maxLength + " got: " + l + " for " + o.getName());
+            }
+        }
+        FileObjectSplit fo = new FileObjectSplit(fileName, mode, array, length, maxLength);
+        return fo;
+    }
+
+    public OutputStream openFileOutputStream(String fileName, boolean append) throws SQLException {
+        fileName = translateFileName(fileName);
+        // TODO the output stream is not split
+        return getFileSystem(fileName).openFileOutputStream(fileName, append);
+    }
+
+    public void rename(String oldName, String newName) throws SQLException {
+        oldName = translateFileName(oldName);
+        newName = translateFileName(newName);
+        for (int i = 0;; i++) {
+            String o = getFileName(oldName, i);
+            if (getFileSystem(o).exists(o)) {
+                String n = getFileName(newName, i);
+                getFileSystem(n).rename(o, n);
+            } else {
+                break;
+            }
+        }
+    }
+
+    public boolean tryDelete(String fileName) {
+        fileName = translateFileName(fileName);
+        for (int i = 0;; i++) {
+            String f = getFileName(fileName, i);
+            if (getFileSystem(fileName).exists(f)) {
+                boolean ok = getFileSystem(fileName).tryDelete(f);
+                if (!ok) {
+                    return false;
+                }
+            } else {
+                break;
+            }
+        }
+        return true;
+    }
+    
+    private String translateFileName(String fileName) {
+        if (!fileName.startsWith(FileSystem.PREFIX_SPLIT)) {
+            throw Message.getInternalError(fileName + " doesn't start with " + FileSystem.PREFIX_SPLIT);
+        }
+        fileName = fileName.substring(FileSystem.PREFIX_SPLIT.length());
+        if (fileName.length() > 0 && Character.isDigit(fileName.charAt(0))) {
+            int idx = fileName.indexOf(':');
+            String size = fileName.substring(0, idx);
+            try {
+                defaultMaxSize = 1L << Integer.decode(size).intValue();
+                fileName = fileName.substring(idx + 1);
+            } catch (NumberFormatException e) {
+                // ignore
+            }
+        }
+        return fileName;
+    }
+    
+    static String getFileName(String fileName, int id) {
+        if (id > 0) {
+            fileName += "." + id + PART_SUFFIX;
+        }
+        return fileName;
+    }
+    
+    private FileSystem getFileSystem(String fileName) {
+        return FileSystem.getInstance(fileName);
+    }
+
+}

h2/src/test/org/h2/test/unit/TestFileSystem.java

@@ -39,6 +39,9 @@ public class TestFileSystem extends TestBase {
     public void test() throws Exception {
         testDatabaseInMemFileSys();
         testDatabaseInJar();
+        // set default part size to 1 << 10
+        FileSystem.getInstance(FileSystem.PREFIX_SPLIT + "10:" + baseDir + "/fs");
+        testFileSystem(FileSystem.PREFIX_SPLIT + baseDir + "/fs");
         testFileSystem(baseDir + "/fs");
         testFileSystem(FileSystem.PREFIX_MEMORY);
         // testFileSystem("jdbc:h2:mem:fs;TRACE_LEVEL_FILE=3");
@@ -177,8 +180,8 @@ public class TestFileSystem extends TestBase {
             fs.createDirs(fsBase + "/testDir/test");
             assertTrue(fs.isDirectory(fsBase + "/testDir"));
             if (!fsBase.startsWith(FileSystem.PREFIX_DB)) {
-                fs.deleteRecursive("/testDir");
-                assertTrue(!fs.exists("/testDir"));
+                fs.deleteRecursive(fsBase + "/testDir");
+                assertTrue(!fs.exists(fsBase + "/testDir"));
             }
         }
         fs.close();
@@ -234,20 +237,20 @@ public class TestFileSystem extends TestBase {
                 len = (int) Math.min(len, ra.length() - ra.getFilePointer());
                 byte[] b1 = new byte[len];
                 byte[] b2 = new byte[len];
-                f.readFully(b1, 0, len);
-                ra.readFully(b2, 0, len);
+                ra.readFully(b1, 0, len);
+                f.readFully(b2, 0, len);
                 trace("readFully " + len);
                 assertEquals(b1, b2);
                 break;
             }
             case 4: {
                 trace("getFilePointer");
-                assertEquals(f.getFilePointer(), ra.getFilePointer());
+                assertEquals(ra.getFilePointer(), f.getFilePointer());
                 break;
             }
             case 5: {
                 trace("length " + ra.length());
-                assertEquals(f.length(), ra.length());
+                assertEquals(ra.length(), f.length());
                 break;
             }
             case 6: {