Remove anon from legacyAlgorithms when creating ssl server sockets.

Restores the ability of H2 in default configuration to use ssl
connections out-of-the-box.

The security property jdk.tls.legacyAlgorithms exists in newer Java
versions and its default value effectively prevents anonymous TLS
from being used. The modification is only attempted when
h2.enableAnonymousTLS is true and makes SSL server socket behaviour
during handshake more similar to that of older JREs.

Note that the setting affects all SSL server sockets on the same JVM.

h2/src/docsrc/html/changelog.html

@@ -21,6 +21,8 @@ Change Log
 
 <h2>Next Version (unreleased)</h2>
 <ul>
+<li>Issue #235: Anonymous SSL connections fail in many situations
+</li>
 <li>Improve performance of cleaning up temp tables - patch from Eric Faulhaber.
 </li>
 <li>Fix bug where table locks were not dropped when the connection closed

h2/src/main/org/h2/security/CipherFactory.java

@@ -17,6 +17,7 @@ import java.net.Socket;
 import java.security.KeyFactory;
 import java.security.KeyStore;
 import java.security.PrivateKey;
+import java.security.Security;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateFactory;
 import java.security.spec.PKCS8EncodedKeySpec;
@@ -24,7 +25,10 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Properties;
+
 import javax.net.ServerSocketFactory;
 import javax.net.ssl.SSLServerSocket;
 import javax.net.ssl.SSLServerSocketFactory;
@@ -48,6 +52,18 @@ public class CipherFactory {
      */
     public static final String KEYSTORE_PASSWORD =
             "h2pass";
+    /**
+     * The security property which can prevent anonymous TLS connections.
+     * Introduced into Java 6,7,8 in updates from July 2015.
+     */
+    public static final String LEGACY_ALGORITHMS_SECURITY_KEY =
+            "jdk.tls.legacyAlgorithms";
+    /**
+     * The value of {@value #LEGACY_ALGORITHMS_SECURITY_KEY} security
+     * property at the time of class initialization.
+     * Null if it is not set.
+     */
+    public static final String DEFAULT_LEGACY_ALGORITHMS = getLegacyAlgoritmsSilently();
 
     private static final String KEYSTORE =
             "~/.h2.keystore";
@@ -56,6 +72,7 @@ public class CipherFactory {
     private static final String KEYSTORE_PASSWORD_KEY =
             "javax.net.ssl.keyStorePassword";
 
+
     private CipherFactory() {
         // utility class
     }
@@ -106,8 +123,13 @@ public class CipherFactory {
     }
 
 /**
-     * Create a secure server socket. If a bind address is specified, the socket
-     * is only bound to this address.
+     * Create a secure server socket. If a bind address is specified, the
+     * socket is only bound to this address.
+     * If h2.enableAnonymousTLS is true, an attempt is made to modify
+     * the security property jdk.tls.legacyAlgorithms (in newer JVMs) to allow
+     * anonymous TLS. This system change is effectively permanent for the
+     * lifetime of the JVM.
+     * @see #removeAnonFromLegacyAlgorithms()
      *
      * @param port the port to listen on
      * @param bindAddress the address to bind to, or null to bind to all
@@ -117,6 +139,9 @@ public class CipherFactory {
     public static ServerSocket createServerSocket(int port,
             InetAddress bindAddress) throws IOException {
         ServerSocket socket = null;
+        if (SysProperties.ENABLE_ANONYMOUS_TLS) {
+            removeAnonFromLegacyAlgorithms();
+        }
         setKeystore();
         ServerSocketFactory f = SSLServerSocketFactory.getDefault();
         SSLServerSocket secureSocket;
@@ -138,6 +163,95 @@ public class CipherFactory {
         return socket;
     }
 
+    /**
+     * Removes DH_anon and ECDH_anon from a comma separated list of ciphers.
+     * Only the first occurrence is removed.
+     * If there is nothing to remove, returns the reference to the argument.
+     * @param commaSepList  a list of names separated by commas (and spaces)
+     * @return  a new string without DH_anon and ECDH_anon items,
+     *          or the original if none were found
+     */
+    public static String removeDhAnonFromCommaSepList(String commaSepList) {
+        if (commaSepList == null) {
+            return commaSepList;
+        }
+        List<String> algos = new LinkedList<String>(Arrays.asList(commaSepList.split("\\s*,\\s*")));
+        boolean dhAnonRemoved = algos.remove("DH_anon");
+        boolean ecdhAnonRemoved = algos.remove("ECDH_anon");
+        if (dhAnonRemoved || ecdhAnonRemoved) {
+            String algosStr = Arrays.toString(algos.toArray(new String[algos.size()]));
+            return (algos.size() > 0) ? algosStr.substring(1, algosStr.length() - 1): "";
+        } else {
+            return commaSepList;
+        }
+    }
+
+    /**
+     * Attempts to weaken the security properties to allow anonymous TLS.
+     * New JREs would not choose an anonymous cipher suite in a TLS handshake
+     * if server-side security property
+     * {@value #LEGACY_ALGORITHMS_SECURITY_KEY}
+     * were not modified from the default value.
+     * <p>
+     * NOTE: In current (as of 2016) default implementations of JSSE which use
+     * this security property, the value is permanently cached inside the
+     * ServerHandshake class upon its first use.
+     * Therefore the modification accomplished by this method has to be done
+     * before the first use of a server SSL socket.
+     * Later changes to this property will not have any effect on server socket
+     * behavior.
+     */
+    public static synchronized void removeAnonFromLegacyAlgorithms() {
+        String legacyAlgosOrig = getLegacyAlgoritmsSilently();
+        if (legacyAlgosOrig == null) {
+            return;
+        }
+        String legacyAlgosNew = removeDhAnonFromCommaSepList(legacyAlgosOrig);
+        if (!legacyAlgosOrig.equals(legacyAlgosNew)) {
+            setLegacyAlgorithmsSilently(legacyAlgosNew);
+        }
+    }
+
+    /**
+     * Attempts to resets the security property to the default value.
+     * The default value of {@value #LEGACY_ALGORITHMS_SECURITY_KEY} was
+     * obtained at time of class initialization.
+     * <p>
+     * NOTE: Resetting the property might not have any effect on server
+     * socket behavior.
+     * @see #removeAnonFromLegacyAlgorithms()
+     */
+    public static synchronized void resetDefaultLegacyAlgorithms() {
+        setLegacyAlgorithmsSilently(DEFAULT_LEGACY_ALGORITHMS);
+    }
+
+    /**
+     * Returns the security property {@value #LEGACY_ALGORITHMS_SECURITY_KEY}.
+     * Ignores security exceptions.
+     * @return  the value of the security property, or null if not set
+     *          or not accessible
+     */
+    public static String getLegacyAlgoritmsSilently() {
+        String defaultLegacyAlgorithms = null;
+        try {
+            defaultLegacyAlgorithms = Security.getProperty(LEGACY_ALGORITHMS_SECURITY_KEY);
+        } catch (SecurityException e) {
+            // ignore
+        }
+        return defaultLegacyAlgorithms;
+    }
+
+    private static void setLegacyAlgorithmsSilently(String legacyAlgos) {
+        if (legacyAlgos == null) {
+            return;
+        }
+        try {
+            Security.setProperty(LEGACY_ALGORITHMS_SECURITY_KEY, legacyAlgos);
+        } catch (SecurityException e) {
+            // ignore
+        }
+    }
+
     private static byte[] getKeyStoreBytes(KeyStore store, String password)
             throws IOException {
         ByteArrayOutputStream bout = new ByteArrayOutputStream();

h2/src/main/org/h2/util/NetUtils.java

@@ -146,7 +146,12 @@ public class NetUtils {
 
     /**
      * Create a server socket. The system property h2.bindAddress is used if
-     * set.
+     * set. If SSL is used and h2.enableAnonymousTLS is true, an attempt is
+     * made to modify the security property jdk.tls.legacyAlgorithms
+     * (in newer JVMs) to allow anonymous TLS.
+     * <p>
+     * This system change is effectively permanent for the lifetime of the JVM.
+     * @see CipherFactory#removeAnonFromLegacyAlgorithms()
      *
      * @param port the port to listen on
      * @param ssl if SSL should be used

h2/src/test/org/h2/test/unit/TestSecurity.java

@@ -9,6 +9,7 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.Arrays;
+
 import org.h2.security.BlockCipher;
 import org.h2.security.CipherFactory;
 import org.h2.security.SHA256;
@@ -35,6 +36,8 @@ public class TestSecurity extends TestBase {
         testSHA();
         testAES();
         testBlockCiphers();
+        testRemoveAnonFromLegacyAlgos();
+        //testResetLegacyAlgos();
     }
 
     private static void testConnectWithHash() throws SQLException {
@@ -251,4 +254,43 @@ public class TestSecurity extends TestBase {
         return len * r < len * 120;
     }
 
+    private void testRemoveAnonFromLegacyAlgos() {
+        String legacyAlgos = "K_NULL, C_NULL, M_NULL, DHE_DSS_EXPORT" +
+                ", DHE_RSA_EXPORT, DH_anon_EXPORT, DH_DSS_EXPORT, DH_RSA_EXPORT, RSA_EXPORT" +
+                ", DH_anon, ECDH_anon, RC4_128, RC4_40, DES_CBC, DES40_CBC";
+        String expectedLegacyAlgosWithoutDhAnon = "K_NULL, C_NULL, M_NULL, DHE_DSS_EXPORT" +
+                ", DHE_RSA_EXPORT, DH_anon_EXPORT, DH_DSS_EXPORT, DH_RSA_EXPORT, RSA_EXPORT" +
+                ", RC4_128, RC4_40, DES_CBC, DES40_CBC";
+        assertEquals(expectedLegacyAlgosWithoutDhAnon,
+                CipherFactory.removeDhAnonFromCommaSepList(legacyAlgos));
+
+        legacyAlgos = "ECDH_anon, DH_anon_EXPORT, DH_anon";
+        expectedLegacyAlgosWithoutDhAnon = "DH_anon_EXPORT";
+        assertEquals(expectedLegacyAlgosWithoutDhAnon,
+                CipherFactory.removeDhAnonFromCommaSepList(legacyAlgos));
+
+        legacyAlgos = null;
+        assertNull(CipherFactory.removeDhAnonFromCommaSepList(legacyAlgos));
+    }
+
+    /**
+     * This test is meaningful when run in isolation. However, tests of server
+     * sockets or ssl connections may modify the global state given by the
+     * jdk.tls.legacyAlgorithms security property (for a good reason).
+     * It is best to avoid running it in test suites, as it could itself lead
+     * to a modification of the global state with hard-to-track consequences.
+     */
+    @SuppressWarnings("unused")
+    private void testResetLegacyAlgos() {
+        String legacyAlgorithmsBefore = CipherFactory.getLegacyAlgoritmsSilently();
+        assertEquals("Failed assumption: jdk.tls.legacyAlgorithms" +
+                " has been modified from its initial setting",
+                CipherFactory.DEFAULT_LEGACY_ALGORITHMS, legacyAlgorithmsBefore);
+        CipherFactory.removeAnonFromLegacyAlgorithms();
+        CipherFactory.resetDefaultLegacyAlgorithms();
+        String legacyAlgorithmsAfter = CipherFactory.getLegacyAlgoritmsSilently();
+        assertEquals(CipherFactory.DEFAULT_LEGACY_ALGORITHMS, legacyAlgorithmsAfter);
+    }
+
+
 }