On out of disk space, the database could get corrupt sometimes, if later write…

On out of disk space, the database could get corrupt sometimes, if later write operations succeeded. Now the file is closed on the first unsuccessful write operation, so that later requests fail consistently. DatabaseEventListener.diskSpaceIsLow() is no longer supported because it can't be guaranteed that it always works correctly.

h2/src/docsrc/html/changelog.html

@@ -18,7 +18,14 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>Csv.write now formats date, time, and timestamp values using java.sql.Date / Time / Timestamp.toString().
+<ul><li>On out of disk space, the database could get corrupt sometimes,
+    if later write operations succeeded. Now the file is closed on the first unsuccessful
+    write operation, so that later requests fail consistently.
+</li><li>DatabaseEventListener.diskSpaceIsLow() is no longer supported because
+    it can't be guaranteed that it always works correctly.
+</li><li>XMLTEXT now supports an optional parameter to escape newlines.
+</li><li>XMLNODE now support an optional parameter to disable indentation.
+</li><li>Csv.write now formats date, time, and timestamp values using java.sql.Date / Time / Timestamp.toString().
     Previously, ResultSet.getString() was used, which didn't work well for Oracle.
 </li><li>The shell script <code>h2.sh</code> can now be run from within a different directory.
     Thanks a lot to Daniel Serodio for the patch!

h2/src/main/org/h2/api/DatabaseEventListener.java

@@ -60,13 +60,6 @@ public interface DatabaseEventListener extends EventListener {
      */
     void opened();
 
-    /**
-     * This method is called if the disk space is very low. One strategy is to
-     * inform the user and wait for it to clean up disk space. The database
-     * should not be accessed from within this method (even to close it).
-     */
-    void diskSpaceIsLow();
-
     /**
      * This method is called if an exception occurred.
      *

h2/src/main/org/h2/command/dml/ScriptBase.java

@@ -197,10 +197,6 @@ abstract class ScriptBase extends Prepared implements DataHandler {
         session.getDatabase().checkWritingAllowed();
     }
 
-    public void freeUpDiskSpace() {
-        session.getDatabase().freeUpDiskSpace();
-    }
-
     public int getMaxLengthInplaceLob() {
         return session.getDatabase().getMaxLengthInplaceLob();
     }

h2/src/main/org/h2/engine/Database.java

@@ -1797,12 +1797,6 @@ public class Database implements DataHandler {
         }
     }
 
-    public synchronized void freeUpDiskSpace() {
-        if (eventListener != null) {
-            eventListener.diskSpaceIsLow();
-        }
-    }
-
     /**
      * Set the progress of a long running operation.
      * This method calls the {@link DatabaseEventListener} if one is registered.

h2/src/main/org/h2/engine/SessionRemote.java

@@ -614,10 +614,6 @@ public class SessionRemote extends SessionWithState implements DataHandler {
         // ok
     }
 
-    public void freeUpDiskSpace() {
-        // nothing to do
-    }
-
     public String getDatabasePath() {
         return "";
     }

h2/src/main/org/h2/store/DataHandler.java

@@ -49,14 +49,6 @@ public interface DataHandler {
      */
     void checkWritingAllowed() throws DbException;
 
-    /**
-     * Free up disk space if possible.
-     * This method is called if more space is needed.
-     *
-     * @throws DbException if no more space could be freed
-     */
-    void freeUpDiskSpace() throws DbException;
-
     /**
      * Get the maximum length of a in-place large object
      *

h2/src/main/org/h2/store/FileStore.java

@@ -326,28 +326,13 @@ public class FileStore {
         try {
             FileUtils.writeFully(file, ByteBuffer.wrap(b, off, len));
         } catch (IOException e) {
-            if (freeUpDiskSpace()) {
-                try {
-                    FileUtils.writeFully(file, ByteBuffer.wrap(b, off, len));
-                } catch (IOException e2) {
-                    throw DbException.convertIOException(e2, name);
-                }
-            } else {
+            closeFileSilently();
             throw DbException.convertIOException(e, name);
         }
-        }
         filePos += len;
         fileLength = Math.max(filePos, fileLength);
     }
 
-    private boolean freeUpDiskSpace() {
-        if (handler == null) {
-            return false;
-        }
-        handler.freeUpDiskSpace();
-        return true;
-    }
-
     /**
      * Set the length of the file. This will expand or shrink the file.
      *
@@ -370,17 +355,10 @@ public class FileStore {
             }
             fileLength = newLength;
         } catch (IOException e) {
-            if (newLength > fileLength && freeUpDiskSpace()) {
-                try {
-                    file.truncate(newLength);
-                } catch (IOException e2) {
-                    throw DbException.convertIOException(e2, name);
-                }
-            } else {
+            closeFileSilently();
             throw DbException.convertIOException(e, name);
         }
     }
-    }
 
     /**
      * Get the file size in bytes.
@@ -434,6 +412,7 @@ public class FileStore {
         try {
             file.force(true);
         } catch (IOException e) {
+            closeFileSilently();
             throw DbException.convertIOException(e, name);
         }
     }
@@ -463,6 +442,19 @@ public class FileStore {
         file = null;
     }
 
+    /**
+     * Just close the file, without setting the reference to null. This method
+     * is called when writing failed. The reference is not set to null so that
+     * there are no NullPointerExceptions later on.
+     */
+    private void closeFileSilently() {
+        try {
+            file.close();
+        } catch (IOException e) {
+            // ignore
+        }
+    }
+
     /**
      * Re-open the file. The file pointer will be reset to the previous
      * location.

h2/src/main/org/h2/tools/Recover.java

@@ -1363,13 +1363,6 @@ public class Recover extends Tool implements DataHandler {
         // nothing to do
     }
 
-    /**
-     * INTERNAL
-     */
-    public void freeUpDiskSpace() {
-        // nothing to do
-    }
-
     /**
      * INTERNAL
      */

h2/src/test/org/h2/test/TestAll.java

@@ -103,6 +103,7 @@ import org.h2.test.server.TestWeb;
 import org.h2.test.server.TestInit;
 import org.h2.test.synth.TestBtreeIndex;
 import org.h2.test.synth.TestCrashAPI;
+import org.h2.test.synth.TestDiskFull;
 import org.h2.test.synth.TestFuzzOptimizations;
 import org.h2.test.synth.TestHaltApp;
 import org.h2.test.synth.TestJoin;
@@ -643,6 +644,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
 
         // synth
         new TestBtreeIndex().runTest(this);
+        new TestDiskFull().runTest(this);
         new TestCrashAPI().runTest(this);
         new TestFuzzOptimizations().runTest(this);
         new TestLimit().runTest(this);

h2/src/test/org/h2/test/synth/TestDiskFull.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.synth;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.h2.constant.ErrorCode;
+import org.h2.test.TestBase;
+import org.h2.test.utils.FilePathUnstable;
+
+/**
+ * Test simulated disk full problems.
+ */
+public class TestDiskFull extends TestBase {
+
+    private FilePathUnstable fs;
+
+    /**
+     * Run just this test.
+     *
+     * @param a ignored
+     */
+    public static void main(String... a) throws Exception {
+        TestBase.createCaller().init().test();
+    }
+
+    public void test() throws Exception {
+        fs = FilePathUnstable.register();
+        test(Integer.MAX_VALUE);
+        int max = Integer.MAX_VALUE - fs.getDiskFullCount() + 10;
+        for (int i = 0; i < max; i++) {
+            test(i);
+        }
+    }
+
+    private boolean test(int x) throws SQLException {
+        deleteDb("memFS:", null);
+        fs.setDiskFullCount(x);
+        String url = "jdbc:h2:unstable:memFS:diskFull;FILE_LOCK=NO;TRACE_LEVEL_FILE=0;WRITE_DELAY=10;LOCK_TIMEOUT=100;CACHE_SIZE=4096";
+        Connection conn = null;
+        Statement stat = null;
+        boolean opened = false;
+        try {
+            conn = DriverManager.getConnection(url);
+            stat = conn.createStatement();
+            opened = true;
+            for (int j = 0; j < 5; j++) {
+                stat.execute("create table test(id int primary key, name varchar)");
+                stat.execute("insert into test values(1, 'Hello')");
+                stat.execute("create index idx_name on test(name)");
+                stat.execute("insert into test values(2, 'World')");
+                stat.execute("update test set name='Hallo' where id=1");
+                stat.execute("delete from test where id=2");
+                stat.execute("checkpoint");
+                stat.execute("insert into test values(3, space(10000))");
+                stat.execute("update test set name='Hallo' where id=3");
+                stat.execute("drop table test");
+            }
+            conn.close();
+            conn = null;
+            return fs.getDiskFullCount() > 0;
+        } catch (SQLException e) {
+            if (e.getErrorCode() == ErrorCode.TABLE_OR_VIEW_ALREADY_EXISTS_1) {
+                throw e;
+            }
+            if (stat != null) {
+                try {
+                    fs.setDiskFullCount(0);
+                    stat.execute("insert into test values(4, space(10000))");
+                    stat.execute("update test set name='Hallo' where id=3");
+                    conn.close();
+                } catch (SQLException e2) {
+                    if (e2.getErrorCode() != ErrorCode.IO_EXCEPTION_1
+                            && e2.getErrorCode() != ErrorCode.IO_EXCEPTION_2) {
+                        throw e2;
+                    }
+                }
+            }
+        } finally {
+            if (conn != null) {
+                try {
+                    if (stat != null) {
+                        stat.execute("shutdown immediately");
+                    }
+                } catch (Exception e2) {
+                    // ignore
+                }
+                try {
+                    conn.close();
+                } catch (Exception e2) {
+                    // ignore
+                }
+            }
+        }
+        fs.setDiskFullCount(0);
+        try {
+            conn = DriverManager.getConnection(url);
+        } catch (SQLException e) {
+            if (!opened) {
+                return false;
+            }
+        }
+        stat = conn.createStatement();
+        stat.execute("script to 'memFS:test.sql'");
+        conn.close();
+        return false;
+    }
+
+}

h2/src/test/org/h2/test/unit/TestDataPage.java

@@ -295,10 +295,6 @@ public class TestDataPage extends TestBase implements DataHandler {
         // ok
     }
 
-    public void freeUpDiskSpace() {
-        // nothing to do
-    }
-
     public int getMaxLengthInplaceLob() {
         throw new AssertionError();
     }

h2/src/test/org/h2/test/unit/TestFile.java

@@ -139,10 +139,6 @@ public class TestFile extends TestBase implements DataHandler {
         // nothing to do
     }
 
-    public void freeUpDiskSpace() {
-        // nothing to do
-    }
-
     public String getDatabasePath() {
         return null;
     }

h2/src/test/org/h2/test/unit/TestValueHashMap.java

@@ -125,10 +125,6 @@ public class TestValueHashMap extends TestBase implements DataHandler {
         // nothing to do
     }
 
-    public void freeUpDiskSpace() {
-        // nothing to do
-    }
-
     public int getMaxLengthInplaceLob() {
         return 0;
     }

h2/src/test/org/h2/test/unit/TestValueMemory.java

@@ -218,10 +218,6 @@ public class TestValueMemory extends TestBase implements DataHandler {
         // nothing to do
     }
 
-    public void freeUpDiskSpace() {
-        // nothing to do
-    }
-
     public String getDatabasePath() {
         return getBaseDir() + "/valueMemory";
     }

h2/src/test/org/h2/test/utils/FilePathUnstable.java

+/*
+ * Copyright 2004-2011 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.util.List;
+import org.h2.store.fs.FileBase;
+import org.h2.store.fs.FilePath;
+import org.h2.store.fs.FilePathWrapper;
+
+/**
+ * An unstable file system. It is used to simulate file system problems (for
+ * example out of disk space).
+ */
+public class FilePathUnstable extends FilePathWrapper {
+
+    private static final FilePathUnstable INSTANCE = new FilePathUnstable();
+
+    private static final IOException DISK_FULL = new IOException("Disk full");
+
+    private static int diskFullOffCount;
+
+    /**
+     * Register the file system.
+     *
+     * @return the instance
+     */
+    public static FilePathUnstable register() {
+        FilePath.register(INSTANCE);
+        return INSTANCE;
+    }
+
+    /**
+     * Check if the simulated problem occurred.
+     * This call will decrement the countdown.
+     *
+     * @throws IOException if the simulated power failure occurred
+     */
+    void checkError() throws IOException {
+        if (diskFullOffCount == 0) {
+            return;
+        }
+        if (--diskFullOffCount > 0) {
+            return;
+        }
+        if (diskFullOffCount >= -4) {
+            diskFullOffCount--;
+            throw DISK_FULL;
+        }
+    }
+
+    public void createDirectory() {
+        super.createDirectory();
+    }
+
+    public boolean createFile() {
+        return super.createFile();
+    }
+
+    public void delete() {
+        super.delete();
+    }
+
+    public boolean exists() {
+        return super.exists();
+    }
+
+    public String getName() {
+        return super.getName();
+    }
+
+    public long lastModified() {
+        return super.lastModified();
+    }
+
+    public FilePath getParent() {
+        return super.getParent();
+    }
+
+    public boolean isAbsolute() {
+        return super.isAbsolute();
+    }
+
+    public boolean isDirectory() {
+        return super.isDirectory();
+    }
+
+    public boolean canWrite() {
+        return super.canWrite();
+    }
+
+    public boolean setReadOnly() {
+        return super.setReadOnly();
+    }
+
+    public long size() {
+        return super.size();
+    }
+
+    public List<FilePath> newDirectoryStream() {
+        return super.newDirectoryStream();
+    }
+
+    public FilePath toRealPath() {
+        return super.toRealPath();
+    }
+
+    public InputStream newInputStream() throws IOException {
+        return super.newInputStream();
+    }
+
+    public FileChannel open(String mode) throws IOException {
+        return new FileUnstable(this, super.open(mode));
+    }
+
+    public OutputStream newOutputStream(boolean append) {
+        return super.newOutputStream(append);
+    }
+
+    public void moveTo(FilePath newName) {
+        super.moveTo(newName);
+    }
+
+    public FilePath createTempFile(String suffix, boolean deleteOnExit, boolean inTempDir)
+            throws IOException {
+        return super.createTempFile(suffix, deleteOnExit, inTempDir);
+    }
+
+    public void setDiskFullCount(int count) {
+        diskFullOffCount = count;
+    }
+
+    public int getDiskFullCount() {
+        return diskFullOffCount;
+    }
+
+    public String getScheme() {
+        return "unstable";
+    }
+
+}
+
+/**
+ * An file that checks for errors before each write operation.
+ */
+class FileUnstable extends FileBase {
+
+    private final FilePathUnstable file;
+    private final FileChannel channel;
+
+    FileUnstable(FilePathUnstable file, FileChannel channel) {
+        this.file = file;
+        this.channel = channel;
+    }
+
+    public void implCloseChannel() throws IOException {
+        channel.close();
+    }
+
+    public long position() throws IOException {
+        return channel.position();
+    }
+
+    public long size() throws IOException {
+        return channel.size();
+    }
+
+    public int read(ByteBuffer dst) throws IOException {
+        return channel.read(dst);
+    }
+
+    public FileChannel position(long pos) throws IOException {
+        channel.position(pos);
+        return this;
+    }
+
+    public FileChannel truncate(long newLength) throws IOException {
+        checkError();
+        channel.truncate(newLength);
+        return this;
+    }
+
+    public void force(boolean metaData) throws IOException {
+        checkError();
+        channel.force(metaData);
+    }
+
+    public int write(ByteBuffer src) throws IOException {
+        checkError();
+        return channel.write(src);
+    }
+
+    private void checkError() throws IOException {
+        file.checkError();
+    }
+
+    public synchronized FileLock tryLock(long position, long size, boolean shared) throws IOException {
+        return channel.tryLock();
+    }
+
+}
\ No newline at end of file

h2/src/tools/org/h2/build/doc/dictionary.txt

@@ -695,4 +695,5 @@ fork tester jaspa redirection johnny brings gone jooq iciql offline pdo mappings
 pst patadia summertime jalpesh scheme compilable ski takanori dsts kawashima
 kokoci seldom jaros ciphers srcs invectorate noah nfontes fontes recoding
 minecraft videos youtube dataflyer bukkit alessio adamo jacopo angel leon frost
-deserializing
+deserializing eckenfelder daniel serodio dirname semicolons whose stretch 
+stabilize