add deadlock detector that runs every 10 seconds and prints out info

about which tables each thread has locked

h2/src/main/org/h2/engine/Engine.java

@@ -17,6 +17,7 @@ import org.h2.store.FileLock;
 import org.h2.util.MathUtils;
 import org.h2.util.New;
 import org.h2.util.StringUtils;
+import org.h2.util.ThreadDeadlockDetector;
 import org.h2.util.Utils;
 
 /**
@@ -35,6 +36,7 @@ public class Engine implements SessionFactory {
 
     private Engine() {
         // use getInstance()
+    	ThreadDeadlockDetector.init();
     }
 
     public static Engine getInstance() {

h2/src/main/org/h2/mvstore/db/MVTable.java

@@ -11,6 +11,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+
 import org.h2.api.DatabaseEventListener;
 import org.h2.api.ErrorCode;
 import org.h2.command.ddl.Analyze;
@@ -37,6 +38,7 @@ import org.h2.table.Column;
 import org.h2.table.IndexColumn;
 import org.h2.table.Table;
 import org.h2.table.TableBase;
+import org.h2.util.DebuggingThreadLocal;
 import org.h2.util.MathUtils;
 import org.h2.util.New;
 import org.h2.value.DataType;
@@ -56,6 +58,10 @@ public class MVTable extends TableBase {
     private final ConcurrentHashMap<Session, Session> lockSharedSessions =
             new ConcurrentHashMap<Session, Session>();
 
+    public static final DebuggingThreadLocal<String> WAITING_FOR_LOCK = new DebuggingThreadLocal<String>();
+    public static final DebuggingThreadLocal<ArrayList<String>> EXCLUSIVE_LOCKS = new DebuggingThreadLocal<ArrayList<String>>();
+    public static final DebuggingThreadLocal<ArrayList<String>> SHARED_LOCKS = new DebuggingThreadLocal<ArrayList<String>>();
+
     /**
      * The queue of sessions waiting to lock the table. It is a FIFO queue to
      * prevent starvation, since Java's synchronized locking is biased.
@@ -130,11 +136,13 @@ public class MVTable extends TableBase {
                 return true;
             }
             session.setWaitForLock(this, Thread.currentThread());
+            WAITING_FOR_LOCK.set(getName());
             waitingSessions.addLast(session);
             try {
                 doLock1(session, lockMode, exclusive);
             } finally {
                 session.setWaitForLock(null, null);
+                WAITING_FOR_LOCK.remove();
                 waitingSessions.remove(session);
             }
         }
@@ -218,11 +226,19 @@ public class MVTable extends TableBase {
                     traceLock(session, exclusive, "added for");
                     session.addLock(this);
                     lockExclusiveSession = session;
+                    if (EXCLUSIVE_LOCKS.get() == null) {
+                      EXCLUSIVE_LOCKS.set(new ArrayList<String>());
+                    }
+                    EXCLUSIVE_LOCKS.get().add(getName());
                     return true;
                 } else if (lockSharedSessions.size() == 1 &&
                         lockSharedSessions.containsKey(session)) {
                     traceLock(session, exclusive, "add (upgraded) for ");
                     lockExclusiveSession = session;
+                    if (EXCLUSIVE_LOCKS.get() == null) {
+                      EXCLUSIVE_LOCKS.set(new ArrayList<String>());
+                    }
+                    EXCLUSIVE_LOCKS.get().add(getName());
                     return true;
                 }
             }
@@ -244,6 +260,10 @@ public class MVTable extends TableBase {
                     traceLock(session, exclusive, "ok");
                     session.addLock(this);
                     lockSharedSessions.put(session, session);
+                    if (SHARED_LOCKS.get() == null) {
+                    	SHARED_LOCKS.set(new ArrayList<String>());
+                    }
+                    SHARED_LOCKS.get().add(getName());
                 }
                 return true;
             }
@@ -357,10 +377,16 @@ public class MVTable extends TableBase {
             traceLock(s, lockExclusiveSession == s, "unlock");
             if (lockExclusiveSession == s) {
                 lockExclusiveSession = null;
+                if (EXCLUSIVE_LOCKS.get() != null) {
+                  EXCLUSIVE_LOCKS.get().remove(getName());
+                }
             }
             synchronized (getLockSyncObject()) {
                 if (lockSharedSessions.size() > 0) {
                     lockSharedSessions.remove(s);
+                    if (SHARED_LOCKS.get() != null) {
+                    	SHARED_LOCKS.get().remove(getName());
+                    }
                 }
                 if (!waitingSessions.isEmpty()) {
                     getLockSyncObject().notifyAll();

h2/src/main/org/h2/util/DebuggingThreadLocal.java

+/*
+ * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.util;
+
+import java.util.HashMap;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Similar to ThreadLocal, except that it allows it's data to be read from other threads - useful for debugging info.
+ */
+public class DebuggingThreadLocal<T>
+{
+  private final ConcurrentHashMap<Long, T> map = new ConcurrentHashMap<Long, T>();
+
+	public DebuggingThreadLocal() {
+	}
+
+	public void set(T value) {
+		map.put(Thread.currentThread().getId(), value);
+	}
+
+	public void remove() {
+		map.remove(Thread.currentThread().getId());
+	}
+
+	public T get() {
+		return map.get(Thread.currentThread().getId());
+	}
+
+	/**
+	 * @return a HashMap containing a mapping from thread-id to value
+	 */
+	public HashMap<Long,T> getSnapshotOfAllThreads() {
+		return new HashMap<Long,T>(map);
+	}
+
+	public T deepCopy(T value) {
+		return value;
+	}
+
+}

h2/src/main/org/h2/util/ThreadDeadlockDetector.java

+/*
+ * Copyright 2004-2014 H2 Group. Multiple-Licensed under the MPL 2.0,
+ * and the EPL 1.0 (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.util;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.management.LockInfo;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MonitorInfo;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.h2.mvstore.db.MVTable;
+
+/**
+ * Detects deadlocks between threads. Prints out data in the same format as the CTRL-BREAK handler,
+ * but includes information about table locks.
+ */
+public class ThreadDeadlockDetector
+{
+	private static String INDENT = "    ";
+
+	private final ThreadMXBean tmbean;
+
+	private final Timer threadCheck = new Timer("ThreadDeadlockDetector", true/* isDaemon */);
+
+	private static ThreadDeadlockDetector detector = null;
+
+	public synchronized static void init() {
+		if (detector == null) {
+			detector = new ThreadDeadlockDetector();
+		}
+	}
+
+	private ThreadDeadlockDetector() {
+		this.tmbean = ManagementFactory.getThreadMXBean();
+		threadCheck.schedule(new TimerTask() {
+			@Override
+			public void run() {
+				checkForDeadlocks();
+			}
+		}, 10/*delay(ms)*/, 10000/*period(ms)*/);
+	}
+
+	/**
+	 * Checks if any threads are deadlocked. If any, print the thread dump information.
+	 */
+	private void checkForDeadlocks() {
+		long[] tids = tmbean.findDeadlockedThreads();
+		if (tids == null) {
+			return;
+		}
+
+		final StringWriter stringWriter = new StringWriter();
+		final PrintWriter print = new PrintWriter(stringWriter);
+
+		print.println("ThreadDeadlockDetector - deadlock found :");
+		final ThreadInfo[] infos = tmbean.getThreadInfo(tids, true, true);
+		final HashMap<Long,String> mvtableWaitingForLockMap =
+				MVTable.WAITING_FOR_LOCK.getSnapshotOfAllThreads();
+		final HashMap<Long,ArrayList<String>> mvtableExclusiveLocksMap =
+				MVTable.EXCLUSIVE_LOCKS.getSnapshotOfAllThreads();
+		final HashMap<Long,ArrayList<String>> mvtableSharedLocksMap =
+				MVTable.SHARED_LOCKS.getSnapshotOfAllThreads();
+		for (ThreadInfo ti : infos) {
+			printThreadInfo(print, ti);
+			printLockInfo(print, ti.getLockedSynchronizers(),
+					mvtableWaitingForLockMap.get(ti.getThreadId()),
+					mvtableExclusiveLocksMap.get(ti.getThreadId()),
+					mvtableSharedLocksMap.get(ti.getThreadId()));
+		}
+
+		print.flush();
+		// Dump it to system.out in one block, so it doesn't get mixed up with other stuff when we're
+		// using a logging subsystem.
+		System.out.println(stringWriter.getBuffer());
+	}
+
+	private static void printThreadInfo(PrintWriter print, ThreadInfo ti) {
+		// print thread information
+		printThread(print, ti);
+
+		// print stack trace with locks
+		StackTraceElement[] stacktrace = ti.getStackTrace();
+		MonitorInfo[] monitors = ti.getLockedMonitors();
+		for (int i = 0; i < stacktrace.length; i++) {
+			StackTraceElement ste = stacktrace[i];
+			print.println(INDENT + "at " + ste.toString());
+			for (MonitorInfo mi : monitors) {
+				if (mi.getLockedStackDepth() == i) {
+					print.println(INDENT + "  - locked " + mi);
+				}
+			}
+		}
+		print.println();
+	}
+
+	private static void printThread(PrintWriter print, ThreadInfo ti) {
+		print.print("\"" + ti.getThreadName() + "\"" + " Id="
+				+ ti.getThreadId() + " in " + ti.getThreadState());
+		if (ti.getLockName() != null) {
+			print.append(" on lock=" + ti.getLockName());
+		}
+		if (ti.isSuspended()) {
+			print.append(" (suspended)");
+		}
+		if (ti.isInNative()) {
+			print.append(" (running in native)");
+		}
+		print.println();
+		if (ti.getLockOwnerName() != null) {
+			print.println(INDENT + " owned by " + ti.getLockOwnerName() + " Id="
+					+ ti.getLockOwnerId());
+		}
+	}
+
+	private static void printLockInfo(PrintWriter print, LockInfo[] locks,
+			String mvtableWaitingForLock,
+			ArrayList<String> mvtableExclusiveLocks,
+			ArrayList<String> mvtableSharedLocksMap) {
+		print.println(INDENT + "Locked synchronizers: count = " + locks.length);
+		for (LockInfo li : locks) {
+			print.println(INDENT + "  - " + li);
+		}
+		if (mvtableWaitingForLock != null) {
+			print.println(INDENT + "Waiting for table: " + mvtableWaitingForLock);
+		}
+		if (mvtableExclusiveLocks != null) {
+			print.println(INDENT + "Exclusive table locks: count = " + mvtableExclusiveLocks.size());
+			for (String name : mvtableExclusiveLocks) {
+				print.println(INDENT + "  - " + name);
+			}
+		}
+		if (mvtableSharedLocksMap != null) {
+			print.println(INDENT + "Shared table locks: count = " + mvtableSharedLocksMap.size());
+			for (String name : mvtableSharedLocksMap) {
+				print.println(INDENT + "  - " + name);
+			}
+		}
+		print.println();
+	}
+
+}
\ No newline at end of file