Merge branch 'master' into LocalTime

.travis.yml

@@ -6,4 +6,4 @@ jdk:
 
 before_script: cd h2
 
-script: ./build.sh jar
+script: ./build.sh jar testFast

h2/src/main/org/h2/store/LobStorageBackend.java

@@ -456,25 +456,31 @@ public class LobStorageBackend implements LobStorageInterface {
             synchronized (conn.getSession()) {
                 try {
                     init();
-                    long lobId = getNextLobId();
-                    String sql = "INSERT INTO " + LOB_MAP + "(LOB, SEQ, POS, HASH, BLOCK) " +
-                            "SELECT ?, SEQ, POS, HASH, BLOCK FROM " + LOB_MAP + " WHERE LOB = ?";
-                    PreparedStatement prep = prepare(sql);
-                    prep.setLong(1, lobId);
-                    prep.setLong(2, oldLobId);
-                    prep.executeUpdate();
-                    reuse(sql, prep);
-
-                    sql = "INSERT INTO " + LOBS + "(ID, BYTE_COUNT, TABLE) " +
-                            "SELECT ?, BYTE_COUNT, ? FROM " + LOBS + " WHERE ID = ?";
-                    prep = prepare(sql);
-                    prep.setLong(1, lobId);
-                    prep.setLong(2, tableId);
-                    prep.setLong(3, oldLobId);
-                    prep.executeUpdate();
-                    reuse(sql, prep);
-
-                    ValueLobDb v = ValueLobDb.create(type, database, tableId, lobId, null, length);
+                    ValueLobDb v = null;
+                    if(!old.isRecoveryReference()){
+                        long lobId = getNextLobId();
+                        String sql = "INSERT INTO " + LOB_MAP + "(LOB, SEQ, POS, HASH, BLOCK) " +
+                                "SELECT ?, SEQ, POS, HASH, BLOCK FROM " + LOB_MAP + " WHERE LOB = ?";
+                        PreparedStatement prep = prepare(sql);
+                        prep.setLong(1, lobId);
+                        prep.setLong(2, oldLobId);
+                        prep.executeUpdate();
+                        reuse(sql, prep);
+
+                        sql = "INSERT INTO " + LOBS + "(ID, BYTE_COUNT, TABLE) " +
+                                "SELECT ?, BYTE_COUNT, ? FROM " + LOBS + " WHERE ID = ?";
+                        prep = prepare(sql);
+                        prep.setLong(1, lobId);
+                        prep.setLong(2, tableId);
+                        prep.setLong(3, oldLobId);
+                        prep.executeUpdate();
+                        reuse(sql, prep);
+
+                        v = ValueLobDb.create(type, database, tableId, lobId, null, length);
+                    }else{
+                        //Recovery process, no need to copy LOB using normal infrastructure
+                        v = ValueLobDb.create(type, database, tableId, oldLobId, null, length);
+                    }
                     return v;
                 } catch (SQLException e) {
                     throw DbException.convert(e);

h2/src/main/org/h2/tools/Recover.java

@@ -218,8 +218,10 @@ public class Recover extends Tool implements DataHandler {
             long precision) {
         DataHandler h = ((JdbcConnection) conn).getSession().getDataHandler();
         verifyPageStore(h);
-        return ValueLobDb.create(Value.BLOB, h, LobStorageFrontend.TABLE_TEMP,
+        ValueLobDb lob = ValueLobDb.create(Value.BLOB, h, LobStorageFrontend.TABLE_TEMP,
                 lobId, null, precision);
+        lob.setRecoveryReference(true);
+        return lob;
     }
 
     private static void verifyPageStore(DataHandler h) {
@@ -237,8 +239,10 @@ public class Recover extends Tool implements DataHandler {
             long precision) {
         DataHandler h = ((JdbcConnection) conn).getSession().getDataHandler();
         verifyPageStore(h);
-        return ValueLobDb.create(Value.CLOB, h, LobStorageFrontend.TABLE_TEMP,
+        ValueLobDb lob =  ValueLobDb.create(Value.CLOB, h, LobStorageFrontend.TABLE_TEMP,
                 lobId, null, precision);
+        lob.setRecoveryReference(true);
+        return lob;
     }
 
     /**

h2/src/main/org/h2/value/ValueLobDb.java

@@ -54,6 +54,11 @@ public class ValueLobDb extends Value implements Value.ValueClob,
     private final FileStore tempFile;
     private int tableId;
     private int hash;
+    
+    //Arbonaut: 13.07.2016
+    // Fix for recovery tool.
+    
+    private boolean isRecoveryReference = false;
 
     private ValueLobDb(int type, DataHandler handler, int tableId, long lobId,
             byte[] hmac, long precision) {
@@ -664,4 +669,12 @@ public class ValueLobDb extends Value implements Value.ValueClob,
         return new ValueLobDb(type, small, precision);
     }
 
+    
+    public void setRecoveryReference(boolean isRecoveryReference) {
+        this.isRecoveryReference = isRecoveryReference;
+    }
+    
+    public boolean isRecoveryReference() {
+        return isRecoveryReference;
+    }
 }

h2/src/test/org/h2/test/TestAll.java

@@ -114,6 +114,7 @@ import org.h2.test.mvcc.TestMvcc3;
 import org.h2.test.mvcc.TestMvcc4;
 import org.h2.test.mvcc.TestMvccMultiThreaded;
 import org.h2.test.poweroff.TestReorderWrites;
+import org.h2.test.recover.RecoverLobTest;
 import org.h2.test.rowlock.TestRowLocks;
 import org.h2.test.server.TestAutoServer;
 import org.h2.test.server.TestInit;
@@ -858,6 +859,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         addTest(new TestReader());
         addTest(new TestRecovery());
         addTest(new TestScriptReader());
+        addTest(new RecoverLobTest());
         addTest(createTest("org.h2.test.unit.TestServlet"));
         addTest(new TestSecurity());
         addTest(new TestShell());
@@ -872,6 +874,7 @@ kill -9 `jps -l | grep "org.h2.test." | cut -d " " -f 1`
         addTest(new TestValue());
         addTest(new TestValueHashMap());
         addTest(new TestWeb());
+        
 
         runAddedTests();
 

h2/src/test/org/h2/test/db/TestMemoryUsage.java

@@ -11,7 +11,6 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Random;
-
 import org.h2.test.TestBase;
 import org.h2.util.Utils;
 
@@ -88,7 +87,7 @@ public class TestMemoryUsage extends TestBase {
         if (usedNow > used * 1.3) {
             // try to lower memory usage (because it might be wrong)
             // by forcing OOME
-            for (int i = 1024;; i *= 2) {
+            for (int i = 1024; i < (1 >> 31); i *= 2) {
                 try {
                     byte[] oome = new byte[1024 * 1024 * 256];
                     oome[0] = (byte) i;

h2/src/test/org/h2/test/db/TestMultiThread.java

@@ -15,10 +15,13 @@ import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Random;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import org.h2.api.ErrorCode;
+import org.h2.jdbc.JdbcSQLException;
 import org.h2.test.TestAll;
 import org.h2.test.TestBase;
 import org.h2.util.SmallLRUCache;
@@ -354,7 +357,17 @@ public class TestMultiThread extends TestBase implements Runnable {
         }
         // check for exceptions
         for (Future<Void> job : jobs) {
-            job.get();
+            try {
+                job.get();
+            } catch (ExecutionException ex) {
+                // ignore timeout exceptions, happens periodically when the machine is really
+                // busy and it's not the thing we are trying to test
+                if (!(ex.getCause() instanceof JdbcSQLException)
+                        || ((JdbcSQLException) ex.getCause())
+                                .getErrorCode() != ErrorCode.LOCK_TIMEOUT_1) {
+                    throw ex;
+                }
+            }
         }
         executor.shutdown();
         executor.awaitTermination(20, TimeUnit.SECONDS);

h2/src/test/org/h2/test/db/TestOutOfMemory.java

@@ -13,14 +13,12 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Map;
 import java.util.Random;
-
 import org.h2.api.ErrorCode;
 import org.h2.mvstore.MVStore;
 import org.h2.store.fs.FilePath;
 import org.h2.store.fs.FilePathMem;
 import org.h2.store.fs.FileUtils;
 import org.h2.test.TestBase;
-import org.h2.tools.DeleteDbFiles;
 
 /**
  * Tests out of memory situations. The database must not get corrupted, and
@@ -48,33 +46,41 @@ public class TestOutOfMemory extends TestBase {
         FilePath.register(new FilePathMem());
         String fileName = "memFS:" + getTestName();
         MVStore store = MVStore.open(fileName);
-        Map<Integer, byte[]> map = store.openMap("test");
-        Random r = new Random(1);
         try {
-            for (int i = 0; i < 100; i++) {
-                byte[] data = new byte[10 * 1024 * 1024];
-                r.nextBytes(data);
-                map.put(i, data);
+            Map<Integer, byte[]> map = store.openMap("test");
+            Random r = new Random(1);
+            try {
+                for (int i = 0; i < 100; i++) {
+                    byte[] data = new byte[10 * 1024 * 1024];
+                    r.nextBytes(data);
+                    map.put(i, data);
+                }
+                fail();
+            } catch (OutOfMemoryError e) {
+                // expected
+            } catch (IllegalStateException e) {
+                // expected
             }
-            fail();
-        } catch (OutOfMemoryError e) {
-            // expected
-        }
-        try {
+            try {
+                store.close();
+            } catch (IllegalStateException e) {
+                // expected
+            }
+            store.closeImmediately();
+            store = MVStore.open(fileName);
+            map = store.openMap("test");
             store.close();
-            fail();
-        } catch (IllegalStateException e) {
-            // expected
+        } finally {
+            // just in case, otherwise if this test suffers a spurious failure, succeeding tests will too
+            // because they will OOM
+            store.closeImmediately();
+            FileUtils.delete(fileName);
         }
-        store.closeImmediately();
-        store = MVStore.open(fileName);
-        map = store.openMap("test");
-        store.close();
-        FileUtils.delete(fileName);
     }
 
     private void testDatabaseUsingInMemoryFileSystem() throws SQLException {
-        String url = "jdbc:h2:memFS:" + getTestName();
+        String filename = "memFS:" + getTestName();
+        String url = "jdbc:h2:" + filename;
         Connection conn = DriverManager.getConnection(url);
         Statement stat = conn.createStatement();
         try {
@@ -94,7 +100,7 @@ public class TestOutOfMemory extends TestBase {
         stat = conn.createStatement();
         stat.execute("select 1");
         conn.close();
-        DeleteDbFiles.execute("memLZF:", getTestName(), true);
+        FileUtils.delete(filename); // release the static data this test generates
     }
 
     private void testUpdateWhenNearlyOutOfMemory() throws SQLException {

h2/src/test/org/h2/test/jdbc/TestPreparedStatement.java

@@ -24,7 +24,6 @@ import java.sql.Statement;
 import java.sql.Timestamp;
 import java.sql.Types;
 import java.util.UUID;
-
 import org.h2.api.ErrorCode;
 import org.h2.api.Trigger;
 import org.h2.test.TestBase;

h2/src/test/org/h2/test/poweroff/TestReorderWrites.java

@@ -44,79 +44,84 @@ public class TestReorderWrites  extends TestBase {
     private void testMVStore() {
         FilePathReorderWrites fs = FilePathReorderWrites.register();
         String fileName = "reorder:memFS:test.mv";
-        for (int i = 0; i < 1000; i++) {
-            log(i + " --------------------------------");
-            Random r = new Random(i);
-            fs.setPowerOffCountdown(100, i);
-            FileUtils.delete(fileName);
-            MVStore store = new MVStore.Builder().
-                    fileName(fileName).
-                    autoCommitDisabled().open();
-            // store.setRetentionTime(10);
-            Map<Integer, byte[]> map = store.openMap("data");
-            map.put(-1, new byte[1]);
-            store.commit();
-            store.getFileStore().sync();
-            int stop = 4 + r.nextInt(20);
-            log("countdown start");
-            fs.setPowerOffCountdown(stop, i);
-            try {
-                for (int j = 1; j < 100; j++) {
-                    Map<Integer, Integer> newMap = store.openMap("d" + j);
-                    newMap.put(j, j * 10);
-                    int key = r.nextInt(10);
-                    int len = 10 * r.nextInt(1000);
-                    if (r.nextBoolean()) {
-                        map.remove(key);
-                    } else {
-                        map.put(key, new byte[len]);
-                    }
-                    log("op " + j + ": ");
-                    store.commit();
-                    switch (r.nextInt(10)) {
-                    case 0:
-                        log("op compact");
-                        store.compact(100, 10 * 1024);
-                        break;
-                    case 1:
-                        log("op compactMoveChunks");
-                        store.compactMoveChunks();
-                        log("op compactMoveChunks done");
-                        break;
+        try {
+            for (int i = 0; i < 1000; i++) {
+                log(i + " --------------------------------");
+                // this test is not interested in power off failures during initial creation
+                fs.setPowerOffCountdown(0, 0);
+                FileUtils.delete(fileName); // release the static data this test generates
+                MVStore store = new MVStore.Builder().
+                        fileName(fileName).
+                        autoCommitDisabled().open();
+                // store.setRetentionTime(10);
+                Map<Integer, byte[]> map = store.openMap("data");
+                map.put(-1, new byte[1]);
+                store.commit();
+                store.getFileStore().sync();
+                Random r = new Random(i);
+                int stop = 4 + r.nextInt(20);
+                log("countdown start");
+                fs.setPowerOffCountdown(stop, i);
+                try {
+                    for (int j = 1; j < 100; j++) {
+                        Map<Integer, Integer> newMap = store.openMap("d" + j);
+                        newMap.put(j, j * 10);
+                        int key = r.nextInt(10);
+                        int len = 10 * r.nextInt(1000);
+                        if (r.nextBoolean()) {
+                            map.remove(key);
+                        } else {
+                            map.put(key, new byte[len]);
+                        }
+                        log("op " + j + ": ");
+                        store.commit();
+                        switch (r.nextInt(10)) {
+                        case 0:
+                            log("op compact");
+                            store.compact(100, 10 * 1024);
+                            break;
+                        case 1:
+                            log("op compactMoveChunks");
+                            store.compactMoveChunks();
+                            log("op compactMoveChunks done");
+                            break;
+                        }
                     }
+                    // write has to fail at some point
+                    fail();
+                } catch (IllegalStateException e) {
+                    log("stop " + e);
+                    // expected
                 }
-                // write has to fail at some point
-                fail();
-            } catch (IllegalStateException e) {
-                log("stop " + e);
-                // expected
-            }
-            try {
+                try {
+                    store.close();
+                } catch (IllegalStateException e) {
+                    // expected
+                    store.closeImmediately();
+                }
+                log("verify");
+                fs.setPowerOffCountdown(100, 0);
+                if (LOG) {
+                    MVStoreTool.dump(fileName, true);
+                }
+                store = new MVStore.Builder().
+                        fileName(fileName).
+                        autoCommitDisabled().open();
+                map = store.openMap("data");
+                if (!map.containsKey(-1)) {
+                    fail("key not found, size=" + map.size() + " i=" + i);
+                } else {
+                    assertEquals("i=" + i, 1, map.get(-1).length);
+                }
+                for (int j = 0; j < 100; j++) {
+                    Map<Integer, Integer> newMap = store.openMap("d" + j);
+                    newMap.get(j);
+                }
+                map.keySet();
                 store.close();
-            } catch (IllegalStateException e) {
-                // expected
-                store.closeImmediately();
-            }
-            log("verify");
-            fs.setPowerOffCountdown(100, 0);
-            if (LOG) {
-                MVStoreTool.dump(fileName, true);
-            }
-            store = new MVStore.Builder().
-                    fileName(fileName).
-                    autoCommitDisabled().open();
-            map = store.openMap("data");
-            if (!map.containsKey(-1)) {
-                fail("key not found, size=" + map.size() + " i=" + i);
-            } else {
-                assertEquals("i=" + i, 1, map.get(-1).length);
-            }
-            for (int j = 0; j < 100; j++) {
-                Map<Integer, Integer> newMap = store.openMap("d" + j);
-                newMap.get(j);
             }
-            map.keySet();
-            store.close();
+        } finally {
+            FileUtils.delete(fileName); // release the static data this test generates
         }
     }
 
@@ -183,6 +188,7 @@ public class TestReorderWrites  extends TestBase {
         }
         assertTrue(minSize < maxSize);
         assertTrue(minWritten < maxWritten);
+        FileUtils.delete(fileName); // release the static data this test generates
     }
 
 }

h2/src/test/org/h2/test/recover/RecoverLobTest.java

+package org.h2.test.recover;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import org.h2.test.TestBase;
+import org.h2.tools.DeleteDbFiles;
+import org.h2.tools.Recover;
+
+public class RecoverLobTest  extends TestBase {
+
+    public static void main(String... a) throws Exception {
+        TestBase.createCaller().init().test();
+    }
+    
+    @Override
+    public TestBase init() throws Exception {
+        TestBase tb = super.init();
+        config.mvStore=false;
+        return tb;
+    }
+
+    @Override
+    public void test() throws Exception {
+        testRecoverClob();
+    }
+   
+    
+    
+    public void testRecoverClob() throws Exception {
+        DeleteDbFiles.execute(getBaseDir(), "recovery", true);
+        Connection conn = getConnection("recovery");
+        Statement stat = conn.createStatement();
+        stat.execute("create table test(id int, data clob)");
+        stat.execute("insert into test values(1, space(10000))");
+        stat.execute("insert into test values(2, space(20000))");
+        stat.execute("insert into test values(3, space(30000))");
+        stat.execute("insert into test values(4, space(40000))");
+        stat.execute("insert into test values(5, space(50000))");
+        stat.execute("insert into test values(6, space(60000))");
+        stat.execute("insert into test values(7, space(70000))");
+        stat.execute("insert into test values(8, space(80000))");
+        
+        conn.close();
+        Recover.main("-dir", getBaseDir(), "-db", "recovery");
+        DeleteDbFiles.execute(getBaseDir(), "recovery", true);
+        conn = getConnection(
+                "recovery;init=runscript from '" +
+                getBaseDir() + "/recovery.h2.sql'");
+        stat = conn.createStatement();
+        
+        ResultSet rs = stat.executeQuery("select * from test");
+        while(rs.next()){
+            
+            int id = rs.getInt(1);
+            String data = rs.getString(2);
+            
+            assertTrue(data != null);
+            assertTrue(data.length() == 10000 * id);
+            
+        }
+        rs.close();
+        conn.close();
+    }
+
+
+
+}

h2/src/test/org/h2/test/store/TestMVStore.java

@@ -15,7 +15,6 @@ import java.util.Random;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
-
 import org.h2.mvstore.Chunk;
 import org.h2.mvstore.Cursor;
 import org.h2.mvstore.DataUtils;
@@ -829,7 +828,7 @@ public class TestMVStore extends TestBase {
         }
         s.close();
         int[] expectedReadsForCacheSize = {
-                3407, 2590, 1924, 1440, 1111, 956, 918
+                3407, 2590, 1924, 1440, 1330, 956, 918
         };
         for (int cacheSize = 0; cacheSize <= 6; cacheSize += 4) {
             int cacheMB = 1 + 3 * cacheSize;

h2/src/test/org/h2/test/utils/FilePathReorderWrites.java

@@ -138,6 +138,12 @@ public class FilePathReorderWrites extends FilePathWrapper {
         return 45000;
     }
 
+    @Override
+    public void delete() {
+        super.delete();
+        FilePath.get(getBase().toString() + ".copy").delete();
+    }
+
 }
 
 /**
@@ -378,7 +384,7 @@ class FileReorderWrites extends FileBase {
                 channel.truncate(position);
                 return -1;
             }
-            // TODO support the case were part is not written
+            // TODO support the case where part is not written
             int len = channel.write(buffer, position);
             buffer.flip();
             return len;