MVStore: the ObjectDataType comparison method was incorrect if one key was…

MVStore: the ObjectDataType comparison method was incorrect if one key was Serializable and the other was of a common class.

h2/src/docsrc/html/changelog.html

@@ -18,7 +18,9 @@ Change Log
 <h1>Change Log</h1>
 
 <h2>Next Version (unreleased)</h2>
-<ul><li>Recursive queries with many result rows (more than the setting "max_memory_rows") 
+<ul><li>MVStore: the ObjectDataType comparison method was incorrect if one
+    key was Serializable and the other was of a common class.
+</li><li>Recursive queries with many result rows (more than the setting "max_memory_rows") 
     did not work correctly.
 </li><li>The license has changed to MPL 2.0 + EPL 1.0.
 </li><li>MVStore: temporary tables from result sets could survive re-opening a database,

h2/src/main/org/h2/mvstore/type/ObjectDataType.java

@@ -1488,9 +1488,12 @@ public class ObjectDataType implements DataType {
             }
             DataType ta = getType(aObj);
             DataType tb = getType(bObj);
-            if (ta != this && ta == tb) {
+            if (ta != this || tb != this) {
+                if (ta == tb) {
                     return ta.compare(aObj, bObj);
                 }
+                return super.compare(aObj, bObj);
+            }
             // TODO ensure comparable type (both may be comparable but not
             // with each other)
             if (aObj instanceof Comparable) {

h2/src/test/org/h2/test/store/TestObjectDataType.java

@@ -118,6 +118,15 @@ public class TestObjectDataType extends TestBase {
             }
             last = x;
         }
+        Random r = new Random(1);
+        for (int i = 0; i < 1000; i++) {
+            Object x = array[r.nextInt(array.length)];
+            Object y = array[r.nextInt(array.length)];
+            int comp = ot.compare(x, y);
+            if (comp != 0) {
+                assertEquals("x:" + x + " y:" + y, -comp, ot.compare(y, x));
+            }
+        }
     }
 
     private void test(Object last, Object x) {