Sometimes a StackOverflow occurred when checking for deadlock.

h2/src/main/org/h2/table/Table.java

@@ -885,16 +885,21 @@ public abstract class Table extends SchemaObjectBase {
 
     /**
      * Check if a deadlock occurred. This method is called recursively. There is
-     * a circle if the session to be tested has already being visited (i.e., it
-     * is one of the 'clash sessions'). In this case the method must return an
-     * empty object array. Once a deadlock has been detected, the methods must
-     * add the session to the list.
+     * a circle if the session to be tested has already being visited. If this
+     * session is part of the circle (if it is the clash session), the method
+     * must return an empty object array. Once a deadlock has been detected, the
+     * methods must add the session to the list. If this session is not part of
+     * the circle, or if no deadlock is detected, this method returns null.
      *
      * @param session the session to be tested for
-     * @param clash set with sessions already visited, and null when starting verification
-     * @return an object array with the sessions involved in the deadlock
-     */
-    public ObjectArray checkDeadlock(Session session, Set clash) {
+     * @param clash set with sessions already visited, and null when starting
+     *            verification
+     * @param visited set with sessions already visited, and null when starting
+     *            verification
+     * @return an object array with the sessions involved in the deadlock, or
+     *         null
+     */
+    public ObjectArray checkDeadlock(Session session, Session clash, Set visited) {
         return null;
     }
 

h2/src/main/org/h2/table/TableData.java

@@ -423,7 +423,7 @@ public class TableData extends Table implements RecordReader {
             }
             session.setWaitForLock(this);
             if (checkDeadlock) {
-                ObjectArray sessions = checkDeadlock(session, null);
+                ObjectArray sessions = checkDeadlock(session, null, null);
                 if (sessions != null) {
                     throw Message.getSQLException(ErrorCode.DEADLOCK_1, getDeadlockDetails(sessions));
                 }
@@ -488,17 +488,23 @@ public class TableData extends Table implements RecordReader {
         return buff.toString();
     }
 
-    public ObjectArray checkDeadlock(Session session, Set clash) {
+    public ObjectArray checkDeadlock(Session session, Session clash, Set visited) {
         // only one deadlock check at any given time
         synchronized (TableData.class) {
             if (clash == null) {
                 // verification is started
-                clash = new HashSet();
-            } else if (clash.contains(session)) {
-                // we found a circle
+                clash = session;
+                visited = new HashSet();
+            } else if (clash == session) {
+                // we found a circle where this session is involved
                 return new ObjectArray();
+            } else if (visited.contains(session)) {
+                // we have already checked this session.
+                // there is a circle, but the sessions in the circle need to
+                // find it out themselves
+                return null;
             }
-            clash.add(session);
+            visited.add(session);
             ObjectArray error = null;
             for (Iterator it = lockShared.iterator(); it.hasNext();) {
                 Session s = (Session) it.next();
@@ -508,7 +514,7 @@ public class TableData extends Table implements RecordReader {
                 }
                 Table t = s.getWaitForLock();
                 if (t != null) {
-                    error = t.checkDeadlock(s, clash);
+                    error = t.checkDeadlock(s, clash, visited);
                     if (error != null) {
                         error.add(session);
                         break;
@@ -518,7 +524,7 @@ public class TableData extends Table implements RecordReader {
             if (error == null && lockExclusive != null) {
                 Table t = lockExclusive.getWaitForLock();
                 if (t != null) {
-                    error = t.checkDeadlock(lockExclusive, clash);
+                    error = t.checkDeadlock(lockExclusive, clash, visited);
                     if (error != null) {
                         error.add(session);
                     }

h2/src/test/org/h2/test/db/TestDeadlock.java

@@ -181,7 +181,7 @@ public class TestDeadlock extends TestBase {
         }
         t2.join();
         t3.join();
-        checkDeadlock(1);
+        checkDeadlock();
         c1.commit();
         c2.commit();
         c3.commit();
@@ -226,7 +226,7 @@ public class TestDeadlock extends TestBase {
       }
       t2.join();
       t3.join();
-      checkDeadlock(2);
+      checkDeadlock();
       c1.commit();
       c2.commit();
       c3.commit();
@@ -260,7 +260,7 @@ public class TestDeadlock extends TestBase {
             catchDeadlock(e);
         }
         t1.join();
-        checkDeadlock(1);
+        checkDeadlock();
         c1.commit();
         c2.commit();
         c1.createStatement().execute("DROP TABLE TEST");
@@ -289,19 +289,19 @@ public class TestDeadlock extends TestBase {
             catchDeadlock(e);
         }
         t1.join();
-        checkDeadlock(1);
+        checkDeadlock();
         c1.commit();
         c2.commit();
         c1.createStatement().execute("DROP TABLE T1, T2");
         end();
     }
 
-    private void checkDeadlock(int max) throws SQLException {
+    private void checkDeadlock() throws SQLException {
         assertTrue(lastException != null);
         assertKnownException(lastException);
         assertEquals(ErrorCode.DEADLOCK_1, lastException.getErrorCode());
         SQLException e2 = lastException.getNextException();
-        if (e2 != null && max == 1) {
+        if (e2 != null) {
             // we have two exception, but there should only be one
             SQLException e3 = new SQLException("Expected one exception, got multiple");
             e3.initCause(e2);