Test SSL sockets with anonymous TLS enabled and disabled

h2/src/test/org/h2/test/unit/TestNetUtils.java

@@ -11,19 +11,30 @@ import java.net.Socket;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLServerSocket;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLSocket;
+
+import org.h2.engine.SysProperties;
 import org.h2.test.TestBase;
 import org.h2.util.NetUtils;
 import org.h2.util.Task;
 
 /**
- * Test the network utilities.
+ * Test the network utilities from {@link NetUtils}.
  *
  * @author Sergi Vladykin
+ * @author Tomas Pospichal
  */
 public class TestNetUtils extends TestBase {
 
     private static final int WORKER_COUNT = 10;
     private static final int PORT = 9111;
+    private static final int WAIT_MILLIS = 100;
+    private static final int WAIT_LONGER_MILLIS = 2 * WAIT_MILLIS;
+    private static final String TASK_PREFIX = "ServerSocketThread-";
 
     /**
      * Run just this test.
@@ -36,10 +47,130 @@ public class TestNetUtils extends TestBase {
 
     @Override
     public void test() throws Exception {
+        testAnonymousTlsSession();
+        testTlsSessionWithServerSideAnonymousDisabled();
         testFrequentConnections(true, 100);
         testFrequentConnections(false, 1000);
     }
 
+    /**
+     * With default settings, H2 client SSL socket should be able to connect
+     * to an H2 server SSL socket using an anonymous cipher suite
+     * (no SSL certificate is needed).
+     * @throws Exception
+     */
+    private void testAnonymousTlsSession() throws Exception {
+        assertTrue("Failed assumption: the default value of ENABLE_ANONYMOUS_TLS" +
+                " property should be true", SysProperties.ENABLE_ANONYMOUS_TLS);
+        boolean ssl = true;
+        Task task = null;
+        ServerSocket serverSocket = null;
+        Socket socket = null;
+
+        try {
+            serverSocket = NetUtils.createServerSocket(PORT, ssl);
+            serverSocket.setSoTimeout(WAIT_LONGER_MILLIS);
+            task = createServerSocketTask(serverSocket);
+            task.execute(TASK_PREFIX + "AnonEnabled");
+            Thread.sleep(WAIT_MILLIS);
+            socket = NetUtils.createLoopbackSocket(PORT, ssl);
+            assertTrue("loopback anon socket should be connected", socket.isConnected());
+            SSLSession session = ((SSLSocket) socket).getSession();
+            assertTrue("TLS session should be valid when anonymous TLS is enabled",
+                    session.isValid());
+            // in case of handshake failure:
+            // the cipher suite is the pre-handshake SSL_NULL_WITH_NULL_NULL
+            assertContains(session.getCipherSuite(), "_anon_");
+        } finally {
+            closeSilently(socket);
+            closeSilently(serverSocket);
+            if (task != null) {
+                // SSL server socket should succeed using an anonymous cipher
+                // suite, and not throw javax.net.ssl.SSLHandshakeException
+                assertNull(task.getException());
+                task.join();
+            }
+        }
+    }
+
+    /**
+     * TLS connections (without trusted certificates) should fail if the server
+     * does not allow anonymous TLS.
+     * The global property ENABLE_ANONYMOUS_TLS cannot be modified for the test;
+     * instead, the server socket is altered.
+     * @throws Exception
+     */
+    private void testTlsSessionWithServerSideAnonymousDisabled() throws Exception {
+        boolean ssl = true;
+        Task task = null;
+        ServerSocket serverSocket = null;
+        Socket socket = null;
+        try {
+            serverSocket = NetUtils.createServerSocket(PORT, ssl);
+            serverSocket.setSoTimeout(WAIT_LONGER_MILLIS);
+            // emulate the situation ENABLE_ANONYMOUS_TLS=false on server side
+            String[] defaultCipherSuites = SSLContext.getDefault().getServerSocketFactory()
+                    .getDefaultCipherSuites();
+            ((SSLServerSocket) serverSocket).setEnabledCipherSuites(defaultCipherSuites);
+            task = createServerSocketTask(serverSocket);
+            task.execute(TASK_PREFIX + "AnonDisabled");
+            Thread.sleep(WAIT_MILLIS);
+            socket = NetUtils.createLoopbackSocket(PORT, ssl);
+            assertTrue("loopback socket should be connected", socket.isConnected());
+            // Java 6 API does not have getHandshakeSession() which could
+            // reveal the actual cipher selected in the attempted handshake
+            SSLSession session = ((SSLSocket) socket).getSession();
+            assertFalse("TLS session should be invalid when the server" +
+                    "disables anonymous TLS", session.isValid());
+            // the SSL handshake should fail, because non-anon ciphers require
+            // a trusted certificate
+            assertEquals("SSL_NULL_WITH_NULL_NULL", session.getCipherSuite());
+        } finally {
+            closeSilently(socket);
+            closeSilently(serverSocket);
+            if (task != null) {
+                assertTrue(task.getException() != null);
+                assertEquals(javax.net.ssl.SSLHandshakeException.class.getName(),
+                        task.getException().getClass().getName());
+                assertContains(task.getException().getMessage(), "certificate_unknown");
+                task.join();
+            }
+        }
+    }
+
+    private Task createServerSocketTask(final ServerSocket serverSocket) {
+        Task task = new Task() {
+            
+            @Override
+            public void call() throws Exception {
+                Socket ss = null;
+                try {
+                    ss = serverSocket.accept();
+                    ss.getOutputStream().write(123);
+                } finally {
+                   closeSilently(ss);
+                }
+            }
+        };
+        return task;
+    }
+
+    private void closeSilently(Socket socket) {
+        try {
+            socket.close();
+        } catch (Exception e) {
+            // ignore
+        }
+    }
+
+    private void closeSilently(ServerSocket socket) {
+        try {
+            socket.close();
+        } catch (Exception e) {
+            // ignore
+        }
+    }
+
     private void testFrequentConnections(boolean ssl, int count) throws Exception {
         final ServerSocket serverSocket = NetUtils.createServerSocket(PORT, ssl);
         final AtomicInteger counter = new AtomicInteger(count);
@@ -96,7 +227,7 @@ public class TestNetUtils extends TestBase {
         private final AtomicInteger counter;
         private Exception exception;
 
-        public ConnectWorker(boolean ssl, AtomicInteger counter) {
+        ConnectWorker(boolean ssl, AtomicInteger counter) {
             this.ssl = ssl;
             this.counter = counter;
         }