Add a scalability test to compare scalability between the old engine and the new MVStore engine.

h2/src/test/org/h2/test/bench/BenchB.java

@@ -25,8 +25,9 @@ public class BenchB implements Bench, Runnable {
     private static final int BRANCHES = 1;
     private static final int TELLERS = 10;
     private static final int ACCOUNTS = 100000;
-    private static final int CLIENTS = 10;
 
+    private int noThreads = 10;
+    
     // master data
     private Database database;
     private int transactionPerClient;
@@ -151,11 +152,12 @@ public class BenchB implements Bench, Runnable {
             // UPDATE ACCOUNTS SET ABALANCE=ABALANCE+? WHERE AID=?
             updateAccount.setInt(1, delta);
             updateAccount.setInt(2, account);
+            master.database.update(updateAccount, "UpdateAccounts");
             updateAccount.executeUpdate();
 
             // SELECT ABALANCE FROM ACCOUNTS WHERE AID=?
             selectAccount.setInt(1, account);
-            ResultSet rs = selectAccount.executeQuery();
+            ResultSet rs = master.database.query(selectAccount);
             while (rs.next()) {
                 rs.getInt(1);
             }
@@ -163,19 +165,19 @@ public class BenchB implements Bench, Runnable {
             // UPDATE TELLERS SET TBALANCE=TABLANCE+? WHERE TID=?
             updateTeller.setInt(1, delta);
             updateTeller.setInt(2, teller);
-            updateTeller.executeUpdate();
+            master.database.update(updateTeller, "UpdateTeller");
 
             // UPDATE BRANCHES SET BBALANCE=BBALANCE+? WHERE BID=?
             updateBranch.setInt(1, delta);
             updateBranch.setInt(2, branch);
-            updateBranch.executeUpdate();
+            master.database.update(updateBranch, "UpdateBranch");
 
             // INSERT INTO HISTORY(TID, BID, AID, DELTA) VALUES(?, ?, ?, ?)
             insertHistory.setInt(1, teller);
             insertHistory.setInt(2, branch);
             insertHistory.setInt(3, account);
             insertHistory.setInt(4, delta);
-            insertHistory.executeUpdate();
+            master.database.update(insertHistory, "InsertHistory");
             conn.commit();
         } catch (SQLException e) {
             e.printStackTrace();
@@ -198,9 +200,9 @@ public class BenchB implements Bench, Runnable {
     }
 
     private void processTransactions() throws Exception {
-        Thread[] threads = new Thread[CLIENTS];
-        for (int i = 0; i < CLIENTS; i++) {
-            threads[i] = new Thread(new BenchB(this, i));
+        Thread[] threads = new Thread[noThreads];
+        for (int i = 0; i < noThreads; i++) {
+            threads[i] = new Thread(new BenchB(this, i), "BenchB-" + i);
         }
         for (Thread t : threads) {
             t.start();
@@ -214,4 +216,8 @@ public class BenchB implements Bench, Runnable {
     public String getName() {
         return "BenchB";
     }
+    
+    public void setNoThreads(int noThreads) {
+        this.noThreads = noThreads;
+    }
 }

h2/src/test/org/h2/test/bench/Database.java

@@ -19,7 +19,7 @@ import java.util.ArrayList;
 import java.util.Properties;
 import java.util.Random;
 import java.util.StringTokenizer;
-
+import java.util.concurrent.atomic.AtomicInteger;
 import org.h2.test.TestBase;
 import org.h2.tools.Server;
 import org.h2.util.JdbcUtils;
@@ -30,9 +30,13 @@ import org.h2.util.StringUtils;
  */
 class Database {
 
+    public interface DatabaseParentInterface {
+        boolean isCollect();
+        void trace(String msg);
+    }
     private static final boolean TRACE = true;
 
-    private TestPerformance test;
+    private DatabaseParentInterface test;
     private int id;
     private String name, url, user, password;
     private final ArrayList<String[]> replace = new ArrayList<String[]>();
@@ -44,8 +48,9 @@ class Database {
     private final Random random = new Random(1);
     private final ArrayList<Object[]> results = new ArrayList<Object[]>();
     private int totalTime;
-    private int executedStatements;
-
+    private final AtomicInteger executedStatements = new AtomicInteger(0);
+    private int noThreads;
+    
     private Server serverH2;
     private Object serverDerby;
     private boolean serverHSQLDB;
@@ -150,7 +155,7 @@ class Database {
      * @param dbString the configuration string
      * @return a new database object with the given settings
      */
-    static Database parse(TestPerformance test, int id, String dbString) {
+    static Database parse(DatabaseParentInterface test, int id, String dbString) {
         try {
             StringTokenizer tokenizer = new StringTokenizer(dbString, ",");
             Database db = new Database();
@@ -172,6 +177,12 @@ class Database {
         }
     }
 
+    static Database parse(DatabaseParentInterface test, int id, String dbString, int noThreads) {
+        Database db = parse(test, id, dbString);
+        db.noThreads = noThreads;
+        return db;
+    }
+    
     /**
      * Open a new database connection. This connection must be closed
      * by calling conn.close().
@@ -285,7 +296,7 @@ class Database {
     void end() {
         long time = System.currentTimeMillis() - startTime;
         log(currentAction, "ms", (int) time);
-        if (test.collect) {
+        if (test.isCollect()) {
             totalTime += time;
         }
     }
@@ -312,8 +323,8 @@ class Database {
     void update(PreparedStatement prep, String traceMessage) throws SQLException {
         test.trace(traceMessage);
         prep.executeUpdate();
-        if (test.collect) {
-            executedStatements++;
+        if (test.isCollect()) {
+            executedStatements.incrementAndGet();
         }
     }
 
@@ -325,8 +336,8 @@ class Database {
     void update(String sql) throws SQLException {
         sql = getSQL(sql);
         if (sql.trim().length() > 0) {
-            if (test.collect) {
-                executedStatements++;
+            if (test.isCollect()) {
+                executedStatements.incrementAndGet();
             }
             stat.execute(sql);
         } else {
@@ -395,7 +406,7 @@ class Database {
      * @param value the value
      */
     void log(String action, String scale, int value) {
-        if (test.collect) {
+        if (test.isCollect()) {
             results.add(new Object[] { action, scale, Integer.valueOf(value) });
         }
     }
@@ -413,8 +424,8 @@ class Database {
         // if(time > 100) {
         //     System.out.println("time="+time);
         // }
-        if (test.collect) {
-            executedStatements++;
+        if (test.isCollect()) {
+            executedStatements.incrementAndGet();
         }
         return rs;
     }
@@ -441,7 +452,7 @@ class Database {
      * @return the number of statements
      */
     int getExecutedStatements() {
-        return executedStatements;
+        return executedStatements.get();
     }
 
     /**
@@ -452,5 +463,9 @@ class Database {
     int getId() {
         return id;
     }
+    
+    int getNoThreads() {
+        return noThreads;
+    }
 
 }

h2/src/test/org/h2/test/bench/TestPerformance.java

@@ -27,7 +27,7 @@ import org.h2.util.JdbcUtils;
  * The main controller class of the benchmark application.
  * To run the benchmark, call the main method of this class.
  */
-public class TestPerformance {
+public class TestPerformance implements Database.DatabaseParentInterface {
 
     /**
      * Whether data should be collected.
@@ -253,10 +253,15 @@ public class TestPerformance {
      *
      * @param s the message
      */
-    void trace(String s) {
+    public void trace(String s) {
         if (trace) {
             System.out.println(s);
         }
     }
+    
+    @Override
+    public boolean isCollect() {
+        return collect;
+    }
 
 }

h2/src/test/org/h2/test/bench/TestScalability.java

+/*
+ * Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License, Version
+ * 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html). Initial Developer: H2 Group
+ */
+package org.h2.test.bench;
+
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import org.h2.store.fs.FileUtils;
+import org.h2.test.TestBase;
+import org.h2.util.IOUtils;
+import org.h2.util.JdbcUtils;
+
+/**
+ * Used to compare scalability between the old engine and the new MVStore engine.
+ * Mostly it runs BenchB with various numbers of threads.
+ */
+public class TestScalability implements Database.DatabaseParentInterface {
+
+    /**
+     * Whether data should be collected.
+     */
+    boolean collect;
+
+    /**
+     * The flag used to enable or disable trace messages.
+     */
+    boolean trace;
+
+    /**
+     * This method is called when executing this sample application.
+     * 
+     * @param args the command line parameters
+     */
+    public static void main(String... args) throws Exception {
+        new TestScalability().test();
+    }
+
+    private static Connection getResultConnection() throws SQLException {
+        org.h2.Driver.load();
+        return DriverManager.getConnection("jdbc:h2:data/results");
+    }
+
+    private static void openResults() throws SQLException {
+        Connection conn = null;
+        Statement stat = null;
+        try {
+            conn = getResultConnection();
+            stat = conn.createStatement();
+            stat.execute("CREATE TABLE IF NOT EXISTS RESULTS(TESTID INT, TEST VARCHAR, "
+                    + "UNIT VARCHAR, DBID INT, DB VARCHAR, RESULT VARCHAR)");
+        } finally {
+            JdbcUtils.closeSilently(stat);
+            JdbcUtils.closeSilently(conn);
+        }
+    }
+
+    private void test() throws Exception {
+        final boolean exit = false;
+        FileUtils.deleteRecursive("data", true);
+        final String out = "benchmark.html";
+        final int size = 400;
+
+        ArrayList<Database> dbs = new ArrayList<Database>();
+        int id = 1;
+        final String h2Url = "jdbc:h2:data/test;LOCK_TIMEOUT=10000;LOCK_MODE=3";
+        dbs.add(createDbEntry(id++, "H2", 1, h2Url));
+        dbs.add(createDbEntry(id++, "H2", 10, h2Url));
+        dbs.add(createDbEntry(id++, "H2", 20, h2Url));
+        dbs.add(createDbEntry(id++, "H2", 30, h2Url));
+        dbs.add(createDbEntry(id++, "H2", 40, h2Url));
+        dbs.add(createDbEntry(id++, "H2", 50, h2Url));
+        dbs.add(createDbEntry(id++, "H2", 100, h2Url));
+        
+        final String mvUrl = "jdbc:h2:data/mvtest;LOCK_TIMEOUT=10000;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine";
+        dbs.add(createDbEntry(id++, "MV", 1, mvUrl));
+        dbs.add(createDbEntry(id++, "MV", 10, mvUrl));
+        dbs.add(createDbEntry(id++, "MV", 20, mvUrl));
+        dbs.add(createDbEntry(id++, "MV", 30, mvUrl));
+        dbs.add(createDbEntry(id++, "MV", 40, mvUrl));
+        dbs.add(createDbEntry(id++, "MV", 50, mvUrl));
+        dbs.add(createDbEntry(id++, "MV", 100, mvUrl));
+        
+        final BenchB test = new BenchB();
+        testAll(dbs, test, size);
+        collect = false;
+
+        ArrayList<Object[]> results = dbs.get(0).getResults();
+        Connection conn = null;
+        PreparedStatement prep = null;
+        Statement stat = null;
+        PrintWriter writer = null;
+        try {
+            openResults();
+            conn = getResultConnection();
+            stat = conn.createStatement();
+            prep = conn
+                    .prepareStatement("INSERT INTO RESULTS(TESTID, TEST, UNIT, DBID, DB, RESULT) VALUES(?, ?, ?, ?, ?, ?)");
+            for (int i = 0; i < results.size(); i++) {
+                Object[] res = results.get(i);
+                prep.setInt(1, i);
+                prep.setString(2, res[0].toString());
+                prep.setString(3, res[1].toString());
+                for (Database db : dbs) {
+                    prep.setInt(4, db.getId());
+                    prep.setString(5, db.getName());
+                    Object[] v = db.getResults().get(i);
+                    prep.setString(6, v[2].toString());
+                    prep.execute();
+                }
+            }
+
+            writer = new PrintWriter(new FileWriter(out));
+            ResultSet rs = stat
+                    .executeQuery("CALL '<table><tr><th>Test Case</th><th>Unit</th>' "
+                            + "|| SELECT GROUP_CONCAT('<th>' || DB || '</th>' ORDER BY DBID SEPARATOR '') FROM "
+                            + "(SELECT DISTINCT DBID, DB FROM RESULTS)"
+                            + "|| '</tr>' || CHAR(10) "
+                            + "|| SELECT GROUP_CONCAT('<tr><td>' || TEST || '</td><td>' || UNIT || '</td>' || ( "
+                            + "SELECT GROUP_CONCAT('<td>' || RESULT || '</td>' ORDER BY DBID SEPARATOR '') FROM RESULTS R2 WHERE "
+                            + "R2.TESTID = R1.TESTID) || '</tr>' ORDER BY TESTID SEPARATOR CHAR(10)) FROM "
+                            + "(SELECT DISTINCT TESTID, TEST, UNIT FROM RESULTS) R1" + "|| '</table>'");
+            rs.next();
+            String result = rs.getString(1);
+            writer.println(result);
+        } finally {
+            JdbcUtils.closeSilently(prep);
+            JdbcUtils.closeSilently(stat);
+            JdbcUtils.closeSilently(conn);
+            IOUtils.closeSilently(writer);
+        }
+
+        if (exit) {
+            System.exit(0);
+        }
+    }
+
+    private Database createDbEntry(int id, String namePrefix, int noThreads, String url) {
+        Database db = Database.parse(this, id,
+                namePrefix + "(" + noThreads + "thread), org.h2.Driver, " + url + ", sa, sa", noThreads);
+        return db;
+    }
+
+
+    private void testAll(ArrayList<Database> dbs, BenchB test, int size) throws Exception {
+        for (int i = 0; i < dbs.size(); i++) {
+            if (i > 0) {
+                Thread.sleep(1000);
+            }
+            // calls garbage collection
+            TestBase.getMemoryUsed();
+            Database db = dbs.get(i);
+            System.out.println("Testing the performance of " + db.getName());
+            db.startServer();
+            Connection conn = db.openNewConnection();
+            DatabaseMetaData meta = conn.getMetaData();
+            System.out.println(" " + meta.getDatabaseProductName() + " " + meta.getDatabaseProductVersion());
+            runDatabase(db, test, 1);
+            runDatabase(db, test, 1);
+            collect = true;
+            runDatabase(db, test, size);
+            conn.close();
+            db.log("Executed statements", "#", db.getExecutedStatements());
+            db.log("Total time", "ms", db.getTotalTime());
+            int statPerSec = db.getExecutedStatements() * 1000 / db.getTotalTime();
+            db.log("Statements per second", "#", statPerSec);
+            System.out.println("Statements per second: " + statPerSec);
+            collect = false;
+            db.stopServer();
+        }
+    }
+
+    private static void runDatabase(Database db, BenchB bench, int size) throws Exception {
+        bench.init(db, size);
+        bench.setNoThreads(db.getNoThreads());
+        bench.runTest();
+    }
+
+    /**
+     * Print a message to system out if trace is enabled.
+     * 
+     * @param s the message
+     */
+    @Override
+    public void trace(String s) {
+        if (trace) {
+            System.out.println(s);
+        }
+    }
+
+    @Override
+    public boolean isCollect() {
+        return collect;
+    }
+}

h2/src/test/org/h2/test/bench/test.properties

-db1 = H2, org.h2.Driver, jdbc:h2:data/test;LOCK_TIMEOUT=10000;LOCK_MODE=3, sa, sa
+db1 = H2(embedded), org.h2.Driver, jdbc:h2:data/test;LOCK_TIMEOUT=10000;LOCK_MODE=3, sa, sa
 
 #xdb1 = H2, org.h2.Driver, jdbc:h2:data/test;LOCK_TIMEOUT=10000;LOCK_MODE=3;DEFAULT_TABLE_ENGINE=org.h2.mvstore.db.MVTableEngine, sa, sa
 
@@ -14,7 +14,7 @@ db1 = H2, org.h2.Driver, jdbc:h2:data/test;LOCK_TIMEOUT=10000;LOCK_MODE=3, sa, s
 db2 = HSQLDB, org.hsqldb.jdbcDriver, jdbc:hsqldb:data/test;hsqldb.default_table_type=cached;sql.enforce_size=true, sa
 db3 = Derby, org.apache.derby.jdbc.EmbeddedDriver, jdbc:derby:data/derby;create=true, sa, sa
 
-db4 = H2, org.h2.Driver, jdbc:h2:tcp://localhost/data/testServer;LOCK_TIMEOUT=10000;LOCK_MODE=3, sa, sa
+db4 = H2(tcpserver), org.h2.Driver, jdbc:h2:tcp://localhost/data/testServer;LOCK_TIMEOUT=10000;LOCK_MODE=3, sa, sa
 db5 = HSQLDB, org.hsqldb.jdbcDriver, jdbc:hsqldb:hsql://localhost/xdb, sa
 db6 = Derby, org.apache.derby.jdbc.ClientDriver, jdbc:derby://localhost/data/derbyServer;create=true, sa, sa
 db7 = PostgreSQL, org.postgresql.Driver, jdbc:postgresql:test, sa, sa