Cluster: after a cluster node failed, the second cluster node can now be re-created.

h2/src/main/org/h2/engine/SessionRemote.java

@@ -82,6 +82,7 @@ public class SessionRemote extends SessionWithState implements SessionFactory, D
     private SessionInterface embedded;
     private DatabaseEventListener eventListener;
     private LobStorage lobStorage;
+    private boolean cluster;
 
     public SessionRemote() {
         // nothing to do
@@ -161,6 +162,7 @@ public class SessionRemote extends SessionWithState implements SessionFactory, D
             switchOffAutoCommit.executeUpdate();
             // so we need to switch it on
             autoCommit = true;
+            cluster = true;
         }
     }
 
@@ -292,10 +294,11 @@ public class SessionRemote extends SessionWithState implements SessionFactory, D
         }
         autoReconnect = Boolean.valueOf(ci.getProperty("AUTO_RECONNECT", "false")).booleanValue();
         // AUTO_SERVER implies AUTO_RECONNECT
-        autoReconnect |= Boolean.valueOf(ci.getProperty("AUTO_SERVER", "false")).booleanValue();
-        if (autoReconnect && serverList != null) {
-            throw DbException.getUnsupportedException("autoReconnect && serverList != null");
+        boolean autoServer = Boolean.valueOf(ci.getProperty("AUTO_SERVER", "false")).booleanValue();
+        if (autoServer && serverList != null) {
+            throw DbException.getUnsupportedException("autoServer && serverList != null");
         }
+        autoReconnect |= autoServer;
         if (autoReconnect) {
             String className = ci.getProperty("DATABASE_EVENT_LISTENER");
             if (className != null) {
@@ -390,7 +393,7 @@ public class SessionRemote extends SessionWithState implements SessionFactory, D
      */
     public void removeServer(IOException e, int i, int count) {
         transferList.remove(i);
-        if (autoReconnect(count)) {
+        if (transferList.size() == 0 && autoReconnect(count)) {
             return;
         }
         checkClosed();
@@ -414,7 +417,10 @@ public class SessionRemote extends SessionWithState implements SessionFactory, D
         if (!isClosed()) {
             return false;
         }
-        if (!autoReconnect || !autoCommit) {
+        if (!autoReconnect) {
+            return false;
+        }
+        if (!cluster && !autoCommit) {
             return false;
         }
         if (count > SysProperties.MAX_RECONNECT) {
@@ -504,7 +510,14 @@ public class SessionRemote extends SessionWithState implements SessionFactory, D
             String sql = transfer.readString();
             int errorCode = transfer.readInt();
             String stackTrace = transfer.readString();
-            throw DbException.convert(new JdbcSQLException(message, sql, sqlstate, errorCode, null, stackTrace));
+            JdbcSQLException s = new JdbcSQLException(message, sql, sqlstate, errorCode, null, stackTrace);
+            if (errorCode == ErrorCode.CONNECTION_BROKEN_1) {
+                // allow re-connect
+                IOException e = new IOException(s.toString());
+                e.initCause(s);
+                throw e;
+            }
+            throw DbException.convert(s);
         } else if (status == STATUS_CLOSED) {
             transferList = null;
         } else if (status == STATUS_OK_STATE_CHANGED) {

h2/src/main/org/h2/table/MetaTable.java

@@ -854,7 +854,6 @@ public class MetaTable extends Table {
                     add(rows, "property." + s, SysProperties.getStringSetting(s, ""));
                 }
             }
-            add(rows, "CLUSTER", database.getCluster());
             add(rows, "EXCLUSIVE", database.getExclusiveSession() == null ? "FALSE" : "TRUE");
             add(rows, "MODE", database.getMode().getName());
             add(rows, "MULTI_THREADED", database.isMultiThreaded() ? "1" : "0");

h2/src/main/org/h2/tools/CreateCluster.java

@@ -11,7 +11,6 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.sql.Statement;
-
 import org.h2.util.IOUtils;
 import org.h2.util.JdbcUtils;
 import org.h2.util.Tool;
@@ -120,34 +119,44 @@ public class CreateCluster extends Tool {
             connSource = DriverManager.getConnection(urlSource + ";CLUSTER=''", user, password);
             statSource = connSource.createStatement();
 
-            // enable the exclusive mode, so that other applications
-            // cannot change the data while it is restoring the second database
-            statSource.execute("SET EXCLUSIVE TRUE");
+            // enable the exclusive mode and close other connections,
+            // so that data can't change while restoring the second database
+            statSource.execute("SET EXCLUSIVE 2");
 
-            // backup
-            Script script = new Script();
-            script.setOut(out);
-            OutputStream scriptOut = null;
             try {
-                scriptOut = IOUtils.openFileOutputStream(scriptFile, false);
-                script.process(connSource, scriptOut);
-            } finally {
-                IOUtils.closeSilently(scriptOut);
-            }
 
-            // restore
-            RunScript runScript = new RunScript();
-            runScript.setOut(out);
-            runScript.process(urlTarget, user, password, scriptFile, null, false);
+                // backup
+                Script script = new Script();
+                script.setOut(out);
+                OutputStream scriptOut = null;
+                try {
+                    scriptOut = IOUtils.openFileOutputStream(scriptFile, false);
+                    script.process(connSource, scriptOut);
+                } finally {
+                    IOUtils.closeSilently(scriptOut);
+                }
+
+                // delete the target database and then restore
+                connTarget = DriverManager.getConnection(urlTarget + ";CLUSTER=''", user, password);
+                statTarget = connTarget.createStatement();
+                statTarget.execute("DROP ALL OBJECTS DELETE FILES");
+                connTarget.close();
 
-            connTarget = DriverManager.getConnection(urlTarget, user, password);
-            statTarget = connTarget.createStatement();
+                RunScript runScript = new RunScript();
+                runScript.setOut(out);
+                runScript.process(urlTarget, user, password, scriptFile, null, false);
 
-            // set the cluster to the serverList on both databases
-            statSource.executeUpdate("SET CLUSTER '" + serverList + "'");
-            statTarget.executeUpdate("SET CLUSTER '" + serverList + "'");
+                connTarget = DriverManager.getConnection(urlTarget, user, password);
+                statTarget = connTarget.createStatement();
 
-            statSource.execute("SET EXCLUSIVE FALSE");
+                // set the cluster to the serverList on both databases
+                statSource.executeUpdate("SET CLUSTER '" + serverList + "'");
+                statTarget.executeUpdate("SET CLUSTER '" + serverList + "'");
+            } finally {
+
+                // switch back to the regular mode
+                statSource.execute("SET EXCLUSIVE FALSE");
+            }
         } finally {
             IOUtils.delete(scriptFile);
             JdbcUtils.closeSilently(statSource);

h2/src/test/org/h2/test/db/TestCluster.java

@@ -17,6 +17,7 @@ import org.h2.test.TestBase;
 import org.h2.tools.CreateCluster;
 import org.h2.tools.DeleteDbFiles;
 import org.h2.tools.Server;
+import org.h2.util.JdbcUtils;
 
 /**
  * Test for the cluster feature.
@@ -32,15 +33,84 @@ public class TestCluster extends TestBase {
         TestBase.createCaller().init().test();
     }
 
-    public void test() throws SQLException {
+    public void test() throws Exception {
+        testCreateClusterAtRuntime();
+        testStartStopCluster();
+    }
+
+    private void testCreateClusterAtRuntime() throws SQLException {
         if (config.memory || config.networked || config.cipher != null) {
             return;
         }
+        int port1 = 9191, port2 = 9192;
+        String serverList = "localhost:" + port1 + ",localhost:" + port2;
         deleteFiles();
 
-        // create the master database
+        org.h2.Driver.load();
+        String user = getUser(), password = getPassword();
+        Connection conn;
+        Statement stat;
+        String url1 = "jdbc:h2:tcp://localhost:" + port1 + "/test";
+        String url2 = "jdbc:h2:tcp://localhost:" + port2 + "/test";
+        String urlCluster = "jdbc:h2:tcp://" + serverList + "/test";
+        int len = 10;
+
+        // initialize the database
+        Server n1 = org.h2.tools.Server.createTcpServer("-tcpPort", "" + port1, "-baseDir", baseDir + "/node1").start();
+        conn = DriverManager.getConnection(url1, user, password);
+        stat = conn.createStatement();
+        stat.execute("create table test(id int primary key, name varchar) as " +
+                "select x, 'Data' || x from system_range(0, " + (len - 1) + ")");
+
+        // start the second server
+        Server n2 = org.h2.tools.Server.createTcpServer("-tcpPort", "" + port2 , "-baseDir", baseDir + "/node2").start();
+
+        // copy the database and initialize the cluster
+        CreateCluster.main("-urlSource", url1, "-urlTarget", url2, "-user", user, "-password", password, "-serverList",
+                serverList);
+
+        // check the original connection is closed
+        try {
+            stat.execute("select * from test");
+            fail();
+        } catch (SQLException e) {
+            // expected
+            JdbcUtils.closeSilently(conn);
+        }
+
+        // test the cluster connection
+        Connection connApp = DriverManager.getConnection(urlCluster + ";AUTO_RECONNECT=TRUE", user, password);
+        check(connApp, len, "'" + serverList + "'");
+
+        // stop server 2, and test if only one server is available
+        n2.stop();
+        check(connApp, len, "''");
+
+        // re-create the cluster
+        n2 = org.h2.tools.Server.createTcpServer("-tcpPort", "" + port2, "-baseDir", baseDir + "/node2").start();
+        CreateCluster.main("-urlSource", url1, "-urlTarget", url2, "-user", user, "-password", password, "-serverList",
+                serverList);
+
+        // test the cluster connection
+        check(connApp, len, "'" + serverList + "'");
+
+        n1.stop();
+        n2.stop();
+        deleteFiles();
+    }
+
+    private void testStartStopCluster() throws SQLException {
+        if (config.memory || config.networked || config.cipher != null) {
+            return;
+        }
+        int port1 = 9193, port2 = 9194;
+        String serverList = "localhost:" + port1 + ",localhost:" + port2;
+        deleteFiles();
+
+        // initialize the database
         Connection conn;
         org.h2.Driver.load();
+
         String urlNode1 = getURL("node1/test", true);
         String urlNode2 = getURL("node2/test", true);
         String user = getUser(), password = getPassword();
@@ -59,39 +129,42 @@ public class TestCluster extends TestBase {
         check(conn, len, "''");
         conn.close();
 
+        // copy the database and initialize the cluster
         CreateCluster.main("-urlSource", urlNode1, "-urlTarget",
                 urlNode2, "-user", user, "-password", password, "-serverList",
-                "localhost:9191,localhost:9192");
-        Server n1 = org.h2.tools.Server.createTcpServer("-tcpPort", "9191", "-baseDir", baseDir + "/node1").start();
-        Server n2 = org.h2.tools.Server.createTcpServer("-tcpPort", "9192", "-baseDir", baseDir + "/node2").start();
+                serverList);
 
+        // start both servers
+        Server n1 = org.h2.tools.Server.createTcpServer("-tcpPort", "" + port1, "-baseDir", baseDir + "/node1").start();
+        Server n2 = org.h2.tools.Server.createTcpServer("-tcpPort", "" + port2, "-baseDir", baseDir + "/node2").start();
+
+        // try to connect in standalone mode - should fail
         try {
-            conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:9191/test", user, password);
+            conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:"+port1+"/test", user, password);
             fail("should not be able to connect in standalone mode");
         } catch (SQLException e) {
             assertKnownException(e);
         }
-
         try {
-            DriverManager.getConnection("jdbc:h2:tcp://localhost:9192/test", user, password);
+            DriverManager.getConnection("jdbc:h2:tcp://localhost:"+port2+"/test", user, password);
             fail("should not be able to connect in standalone mode");
         } catch (SQLException e) {
             assertKnownException(e);
         }
 
-        // test regular cluster connection
-        conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:9191,localhost:9192/test", user, password);
-        check(conn, len, "'localhost:9191,localhost:9192'");
+        // test a cluster connection
+        conn = DriverManager.getConnection("jdbc:h2:tcp://" + serverList + "/test", user, password);
+        check(conn, len, "'"+serverList+"'");
         conn.close();
 
-        // test if only one server is available at the beginning
+        // stop server 2, and test if only one server is available
         n2.stop();
-        conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:9191,localhost:9192/test", user, password);
+        conn = DriverManager.getConnection("jdbc:h2:tcp://" + serverList + "/test", user, password);
         check(conn, len, "''");
         conn.close();
 
         // disable the cluster
-        conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:9191/test;CLUSTER=''", user, password);
+        conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:"+port1+"/test;CLUSTER=''", user, password);
         conn.close();
         n1.stop();
 
@@ -99,11 +172,11 @@ public class TestCluster extends TestBase {
         DeleteDbFiles.main("-dir", baseDir + "/node2", "-quiet");
         CreateCluster.main("-urlSource", urlNode1, "-urlTarget",
                 urlNode2, "-user", user, "-password", password, "-serverList",
-                "localhost:9191,localhost:9192");
-        n1 = org.h2.tools.Server.createTcpServer("-tcpPort", "9191", "-baseDir", baseDir + "/node1").start();
-        n2 = org.h2.tools.Server.createTcpServer("-tcpPort", "9192", "-baseDir", baseDir + "/node2").start();
+                serverList);
+        n1 = org.h2.tools.Server.createTcpServer("-tcpPort", "" + port1, "-baseDir", baseDir + "/node1").start();
+        n2 = org.h2.tools.Server.createTcpServer("-tcpPort", "" + port2, "-baseDir", baseDir + "/node2").start();
 
-        conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:9191,localhost:9192/test", user, password);
+        conn = DriverManager.getConnection("jdbc:h2:tcp://" + serverList + "/test", user, password);
         stat = conn.createStatement();
         stat.execute("CREATE TABLE BOTH(ID INT)");
 
@@ -113,14 +186,14 @@ public class TestCluster extends TestBase {
         conn.close();
         n2.stop();
 
-        n1 = org.h2.tools.Server.createTcpServer("-tcpPort", "9191", "-baseDir", baseDir + "/node1").start();
-        conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:9191/test;CLUSTER=''", user, password);
+        n1 = org.h2.tools.Server.createTcpServer("-tcpPort", "" + port1, "-baseDir", baseDir + "/node1").start();
+        conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:"+port1+"/test;CLUSTER=''", user, password);
         check(conn, len, "''");
         conn.close();
         n1.stop();
 
-        n2 = org.h2.tools.Server.createTcpServer("-tcpPort", "9192", "-baseDir", baseDir + "/node2").start();
-        conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:9192/test;CLUSTER=''", user, password);
+        n2 = org.h2.tools.Server.createTcpServer("-tcpPort", "" + port2, "-baseDir", baseDir + "/node2").start();
+        conn = DriverManager.getConnection("jdbc:h2:tcp://localhost:" + port2 + "/test;CLUSTER=''", user, password);
         check(conn, len, "''");
         conn.createStatement().execute("SELECT * FROM A");
         conn.close();