--no commit message

--no commit message

h2/src/main/org/h2/command/dml/TransactionCommand.java

@@ -13,6 +13,7 @@ import org.h2.engine.Session;
 import org.h2.log.LogSystem;
 import org.h2.message.Message;
 import org.h2.result.LocalResult;
+import org.h2.util.TempFileDeleter;
 
 /**
  * Represents a transactional statement.
@@ -66,6 +67,7 @@ public class TransactionCommand extends Prepared {
         case CHECKPOINT:
             session.getUser().checkAdmin();
             session.getDatabase().getLog().checkpoint();
+            TempFileDeleter.deleteUnused();
             break;
         case SAVEPOINT:
             session.addSavepoint(savepointName);

h2/src/main/org/h2/constant/SysProperties.java

@@ -217,7 +217,9 @@ public class SysProperties {
      * more than this number of rows are in a result set, a temporary table is
      * used.
      */
-    public static final int MAX_MEMORY_ROWS_DISTINCT = getIntSetting("h2.maxMemoryRowsDistinct", Integer.MAX_VALUE);
+private int testing;    
+//    public static final int MAX_MEMORY_ROWS_DISTINCT = getIntSetting("h2.maxMemoryRowsDistinct", Integer.MAX_VALUE);
+public static final int MAX_MEMORY_ROWS_DISTINCT = getIntSetting("h2.maxMemoryRowsDistinct", 1);
 
     /**
      * System property <code>h2.maxQueryTimeout</code> (default: 0).<br />

h2/src/main/org/h2/engine/Session.java

@@ -68,7 +68,7 @@ public class Session implements SessionInterface {
     private String currentSchemaName;
     private String[] schemaSearchPath;
     private String traceModuleName;
-    private HashSet unlinkSet;
+    private HashMap unlinkMap;
     private int tempViewIndex;
     private HashMap procedures;
     private static int nextSerialId;
@@ -279,16 +279,16 @@ public class Session implements SessionInterface {
                 autoCommitAtTransactionEnd = false;
             }
         }
-        if (unlinkSet != null && unlinkSet.size() > 0) {
+        if (unlinkMap != null && unlinkMap.size() > 0) {
             // need to flush the log file, because we can't unlink lobs if the
             // commit record is not written
             logSystem.flush();
-            Iterator it = unlinkSet.iterator();
+            Iterator it = unlinkMap.values().iterator();
             while (it.hasNext()) {
                 Value v = (Value) it.next();
                 v.unlink();
             }
-            unlinkSet = null;
+            unlinkMap = null;
         }
         unlockAll();
     }
@@ -635,15 +635,15 @@ public class Session implements SessionInterface {
     }
 
     public void unlinkAtCommit(Value v) {
-        if (unlinkSet == null) {
-            unlinkSet = new HashSet();
+        if (unlinkMap == null) {
+            unlinkMap = new HashMap();
         }
-        unlinkSet.add(v);
+        unlinkMap.put(v.toString(), v);
     }
 
     public void unlinkAtCommitStop(Value v) {
-        if (unlinkSet != null) {
-            unlinkSet.remove(v);
+        if (unlinkMap != null) {
+            unlinkMap.remove(v.toString());
         }
     }
 

h2/src/main/org/h2/index/ScanIndex.java

@@ -104,7 +104,9 @@ public class ScanIndex extends BaseIndex {
                 for (int i = 0; i < row.getColumnCount(); i++) {
                     Value v = row.getValue(i);
                     Value v2 = v.link(database, getId());
+                    if (v2.isLinked()) {
                         session.unlinkAtCommitStop(v2);
+                    }
                     if (v != v2) {
                         row.setValue(i, v2);
                     }

h2/src/main/org/h2/result/LocalResult.java

@@ -218,7 +218,7 @@ public class LocalResult implements ResultInterface {
                 distinctRows.put(array, values);
                 rowCount = distinctRows.size();
                 if (rowCount > SysProperties.MAX_MEMORY_ROWS_DISTINCT) {
-                    disk = new ResultDiskBuffer(session, sort, values.length);
+                    disk = new ResultTempTable(session, sort, values.length);
                     disk.addRows(distinctRows.values());
                     distinctRows = null;
                 }
@@ -231,7 +231,7 @@ public class LocalResult implements ResultInterface {
         rowCount++;
         if (rows.size() > maxMemoryRows && session.getDatabase().isPersistent()) {
             if (disk == null) {
-                disk = new ResultTempTable(session, sort, values.length);
+                disk = new ResultDiskBuffer(session, sort, values.length);
             }
             addRowsToDisk();
         }
@@ -251,6 +251,31 @@ public class LocalResult implements ResultInterface {
             if (distinctRows != null) {
                 rows = distinctRows.values();
                 distinctRows = null;
+            } else {
+                if (disk != null && sort != null) {
+                    // external sort
+                    ResultExternal temp = disk;
+                    disk = null;
+                    temp.reset();
+                    rows = new ObjectArray();
+                    // TODO use offset directly if possible
+                    while (true) {
+                        Value[] list = temp.next();
+                        if (list == null) {
+                            break;
+                        }
+                        if (disk == null) {
+                            disk = new ResultDiskBuffer(session, sort, list.length);
+                        }
+                        rows.add(list);
+                        if (rows.size() > maxMemoryRows) {
+                            disk.addRows(rows);
+                            rows.clear();
+                        }
+                    }
+                    temp.close();
+                    // the remaining data in rows is written in the following lines
+                }
             }
         }
         if (disk != null) {

h2/src/main/org/h2/result/ResultTempTable.java

@@ -9,9 +9,11 @@ import java.sql.SQLException;
 
 import org.h2.engine.Constants;
 import org.h2.engine.Session;
+import org.h2.index.BtreeIndex;
 import org.h2.index.Cursor;
 import org.h2.index.Index;
 import org.h2.index.IndexType;
+import org.h2.message.Message;
 import org.h2.schema.Schema;
 import org.h2.table.Column;
 import org.h2.table.IndexColumn;
@@ -19,7 +21,6 @@ import org.h2.table.TableData;
 import org.h2.util.ObjectArray;
 import org.h2.value.Value;
 import org.h2.value.ValueArray;
-import org.h2.value.ValueUuid;
 
 /**
  * This class implements the temp table buffer for the LocalResult class.
@@ -35,13 +36,13 @@ public class ResultTempTable implements ResultExternal {
     public ResultTempTable(Session session, SortOrder sort, int columnCount) throws SQLException {
         this.session = session;
         this.sort = sort;
-        String tableName = "TEMP_" + ValueUuid.getNewRandom().toString();
         Schema schema = session.getDatabase().getSchema(Constants.SCHEMA_MAIN);
         Column column = new Column(COLUMN_NAME, Value.ARRAY);
         column.setNullable(false);
         ObjectArray columns = new ObjectArray();
         columns.add(column);
         int tableId = session.getDatabase().allocateObjectId(true, true);
+        String tableName = "TEMP_RESULT_SET_" + tableId;
         table = schema.createTable(tableName, tableId, columns, false, false);
         int indexId = session.getDatabase().allocateObjectId(true, true);
         IndexColumn indexColumn = new IndexColumn();
@@ -50,27 +51,30 @@ public class ResultTempTable implements ResultExternal {
         IndexType indexType;
         indexType = IndexType.createPrimaryKey(true, false);
         IndexColumn[] indexCols = new IndexColumn[]{indexColumn};
-        index = table.addIndex(session, tableName, indexId, indexCols, indexType, Index.EMPTY_HEAD, null);
+        index = new BtreeIndex(session, table, indexId, tableName, indexCols, indexType, Index.EMPTY_HEAD);
+        table.getIndexes().add(index);
     }
     
     public int  removeRow(Value[] values) throws SQLException {
-        ValueArray data = ValueArray.get(values);
-        Row row = new Row(new Value[]{data}, data.getMemory());
+        Row row = convertToRow(values);
+        Cursor cursor = find(row);
+        if (cursor != null) {
+            row = cursor.get();
             table.removeRow(session, row);
+        }
         return (int) table.getRowCount(session);
     }
     
     public boolean contains(Value[] values) throws SQLException {
-        ValueArray data = ValueArray.get(values);
-        Row row = new Row(new Value[]{data}, data.getMemory());
-        Cursor cursor = index.find(session, row, row);
-        return cursor.next();
+        return find(convertToRow(values)) != null;
     }
     
     public int addRow(Value[] values) throws SQLException {
-        ValueArray data = ValueArray.get(values);
-        Row row = new Row(new Value[]{data}, data.getMemory());
+        Row row = convertToRow(values);
+        Cursor cursor = find(row);
+        if (cursor == null) {
             table.addRow(session, row);
+        }
         return (int) table.getRowCount(session);
     }
     
@@ -85,13 +89,17 @@ public class ResultTempTable implements ResultExternal {
     }
 
     public void close() {
-        int todo;
         try {
+            if (table != null) {
                 index.remove(session);
+                ObjectArray indexes = table.getIndexes();
+                indexes.remove(indexes.indexOf(index));
                 table.removeChildrenAndResources(session);
+            }
         } catch (SQLException e) {
-            int test;
-            e.printStackTrace();
+            throw Message.getInternalError();
+        } finally {
+            table = null;
         }
     }
 
@@ -110,5 +118,23 @@ public class ResultTempTable implements ResultExternal {
     public void reset() throws SQLException {
         cursor = index.find(session, null, null);
     }
+    
+    private Row convertToRow(Value[] values) {
+        ValueArray data = ValueArray.get(values);
+        return new Row(new Value[]{data}, data.getMemory());
+    }    
+    
+    private Cursor find(Row row) throws SQLException {
+        Cursor cursor = index.find(session, row, row);
+        while (cursor.next()) {
+            SearchRow found;
+            found = cursor.getSearchRow();
+            if (found.getValue(0).equals(row.getValue(0))) {
+                return cursor;
+            }
+        }
+        return null;
+    }
+    
 }
 

h2/src/main/org/h2/util/TempFileDeleter.java

@@ -35,6 +35,7 @@ public class TempFileDeleter {
             if (SysProperties.CHECK && f2 != null && fileName != null && !f2.equals(fileName)) {
                 throw Message.getInternalError("f2:" + f2 + " f:" + fileName);
             }
+            fileName = f2;
         }
         if (fileName != null && FileUtils.exists(fileName)) {
             try {
@@ -43,7 +44,6 @@ public class TempFileDeleter {
             } catch (Exception e) {
                 // TODO log such errors?
             }
-            deleteUnused();
         }
     }
 

h2/src/main/org/h2/value/ValueLob.java

@@ -519,7 +519,7 @@ public class ValueLob extends Value {
             try {
                 hash = ByteUtils.getByteArrayHash(getBytes());
             } catch (SQLException e) {
-                // TODO hash code for lob: should not ignore exception
+                throw Message.convertToInternal(e);
             }
         }
         return hash;
@@ -607,7 +607,6 @@ public class ValueLob extends Value {
     }
 
     public int getDisplaySize() {
-        // TODO display size of lob?
         return MathUtils.convertLongToInt(getPrecision());
     }
 

h2/src/test/org/h2/test/TestAll.java

@@ -159,6 +159,34 @@ java org.h2.test.TestAll timer
 
 /*
 
+disk based select distinct; order by:
+
+DROP TABLE TEST;
+
+CREATE TABLE TEST(ID INT PRIMARY KEY, NAME VARCHAR(255), VALUE DECIMAL(10,2));
+
+INSERT INTO TEST VALUES(1,'Apples',1.20), 
+(2,'Oranges',2.05),
+(3,'Cherries',5.10),
+(4,'Apples',1.50),
+(5,'Apples',1.10),
+(6,'Oranges',1.80),
+(7,'Bananas',2.50),
+(8,NULL,3.10),
+(9,NULL,-10.0);
+
+SELECT DISTINCT NAME FROM TEST;
+
+select distinct x from system_range(1, 200000);
+
+CREATE TABLE p(d DATE);
+INSERT INTO p VALUES('0000-01-01');
+INSERT INTO p VALUES('0001-01-01');
+
+check that test data in temp directories
+
+add jdbcx to Javadocs
+
 out of memory problem:
 java -XX:+HeapDumpOnOutOfMemoryError -Xmx1024m -cp bin/h2.jar org.h2.tools.Server -log true
 jdbc:h2:test;CACHE_SIZE=10000
@@ -169,17 +197,11 @@ CREATE TABLE secondtable (field1 number(10) not null, field2 number(8));
 create index idx_number1 on secondtable(field1);
 The table 'secondtable' contains a little over 100.000.000 records.
 
-disk based select distinct; order by
-set max_memory_rows 10;
-select distinct x from system_range(1, 200000);
-set max_memory_rows 10000;
-
+Read HenPlus features
+http://henplus.sourceforge.net/
 
 newsletter.sql
 
-BlobTestCase2
-H2BlobWriteTester
-
 add link to new in use, links
 
 Add google site search to web page
@@ -208,10 +230,13 @@ Can sometimes not delete log file? need test case
 Add where required // TODO: change in version 1.1
 
 History:
+Very large SELECT DISTINCT and UNION EXCEPT queries are now supported.
 Improved support for IKVM.
 A error is now thrown when trying to call a method
     inside a trigger that implicitly commits the current transaction,
     if an object is locked.
+Unused LOB files were deleted much too late. 
+    Now they are deleted if no longer referenced in memory.
 
 Roadmap:
 

h2/src/test/org/h2/test/TestBase.java

@@ -30,7 +30,11 @@ import org.h2.tools.DeleteDbFiles;
  */
 public abstract class TestBase {
 
-    protected static String baseDir = "data";
+    public static String getTestDir(String name) {
+        return System.getProperty("java.io.tmpdir") + "/dbtest-" + name;
+    }
+    
+    protected static String baseDir = System.getProperty("java.io.tmpdir") + "/dbtest";
 
     protected TestAll config;
     private long start;

h2/src/test/org/h2/test/db/TestLob.java

@@ -19,9 +19,11 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Savepoint;
 import java.sql.Statement;
+import java.util.ArrayList;
 import java.util.Random;
 
 import org.h2.constant.SysProperties;
+import org.h2.store.FileLister;
 import org.h2.test.TestBase;
 import org.h2.util.IOUtils;
 import org.h2.util.StringUtils;
@@ -35,6 +37,7 @@ public class TestLob extends TestBase {
         if (config.memory) {
             return;
         }
+        testLobDelete();
         testLobVariable();
         testLobDrop();
         testLobNoClose();
@@ -56,6 +59,37 @@ public class TestLob extends TestBase {
         testJavaObject();
     }
     
+    private void testLobDelete() throws Exception {
+        if (config.memory) {
+            return;
+        }
+        deleteDb("lob");
+        Connection conn = reconnect(null);
+        Statement stat = conn.createStatement();
+        stat.execute("CREATE TABLE TEST(ID INT, DATA CLOB)");
+        stat.execute("INSERT INTO TEST SELECT X, SPACE(10000) FROM SYSTEM_RANGE(1, 10)");
+        ArrayList list = FileLister.getDatabaseFiles(baseDir, "lob", true);
+        stat.execute("UPDATE TEST SET DATA = SPACE(5000)");
+        for (int i = 0; i < 3; i++) {
+            System.gc();
+        }
+        stat.execute("CHECKPOINT");
+        ArrayList list2 = FileLister.getDatabaseFiles(baseDir, "lob", true);
+        if (list2.size() >= list.size() + 2) {
+            error("Expected not many more files, got " + list2.size() + " was " + list.size());
+        }
+        stat.execute("DELETE FROM TEST");
+        for (int i = 0; i < 3; i++) {
+            System.gc();
+        }
+        stat.execute("CHECKPOINT");
+        ArrayList list3 = FileLister.getDatabaseFiles(baseDir, "lob", true);
+        if (list3.size() >= list.size()) {
+            error("Expected less files, got " + list2.size() + " was " + list.size());
+        }
+        conn.close();
+    }
+
     private void testLobVariable() throws Exception {
         deleteDb("lob");
         Connection conn = reconnect(null);

h2/src/test/org/h2/test/synth/TestBtreeIndex.java

@@ -30,7 +30,7 @@ public class TestBtreeIndex extends TestBase {
     }
 
     public void testCase(int seed) throws Exception {
-        baseDir = "dataIndex";
+        baseDir = TestBase.getTestDir("index");
         testOne(seed);
         baseDir = "data";
     }

h2/src/test/org/h2/test/synth/TestCrashAPI.java

@@ -389,7 +389,7 @@ public class TestCrashAPI extends TestBase {
         if (config.mvcc || config.networked || config.logMode == 0) {
             return this;
         }
-        baseDir = "dataCrash";
+        baseDir = TestBase.getTestDir("crash");
         startServerIfRequired();
         TestScript script = new TestScript();
         ArrayList add = script.getAllStatements(config, "org/h2/test/test.in.txt");

h2/src/test/org/h2/test/synth/TestHalt.java

@@ -38,7 +38,7 @@ public abstract class TestHalt extends TestBase {
     private int errorId;
     private int sequenceId;
     private static final String DATABASE_NAME = "halt";
-    static final String DIR = "dataHalt";
+    static final String DIR = TestBase.getTestDir("halt");
     private static final String TRACE_FILE_NAME = "haltTrace.trace.db";
 
     abstract void testInit() throws Exception;

h2/src/test/org/h2/test/synth/TestJoin.java

@@ -31,9 +31,10 @@ public class TestJoin extends TestBase {
     private StringBuffer buff;
 
     public void test() throws Exception {
-        baseDir = "dataJoin";
+        String old = baseDir;
+        baseDir = TestBase.getTestDir("join");
         testJoin();
-        baseDir = "data";
+        baseDir = old;
     }
 
     private void testJoin() throws Exception {

h2/src/test/org/h2/test/synth/TestKill.java

@@ -25,7 +25,7 @@ public class TestKill extends TestBase {
     Connection conn;
     int accounts = 10;
     Random random = new Random(1);
-    private static final String DIR = "synth";
+    private static final String DIR = TestBase.getTestDir("kill");
 
     public void test() throws Exception {
         String connect = "";

h2/src/test/org/h2/test/synth/sql/TestSynth.java

@@ -217,7 +217,7 @@ public class TestSynth extends TestBase {
 
     public TestBase init(TestAll conf) throws Exception {
         super.init(conf);
-        baseDir = "dataSynth";
+        baseDir = TestBase.getTestDir("synth");
         deleteDb("synth");
         databases = new ArrayList();
 

h2/src/test/org/h2/test/test.in.txt

@@ -6744,6 +6744,23 @@ SELECT DISTINCT NAME FROM TEST;
 > null
 > rows: 5
 
+SELECT DISTINCT NAME FROM TEST ORDER BY NAME DESC NULLS LAST;
+> NAME
+> --------
+> Oranges
+> Cherries
+> Bananas
+> Apples
+> null
+> rows (ordered): 5
+
+SELECT DISTINCT NAME FROM TEST ORDER BY NAME DESC NULLS LAST LIMIT 2 OFFSET 1;
+> NAME
+> --------
+> Cherries
+> Bananas
+> rows (ordered): 2
+
 SELECT NAME, COUNT(*), SUM(VALUE), MAX(VALUE), MIN(VALUE), AVG(VALUE), COUNT(DISTINCT VALUE) FROM TEST GROUP BY NAME;
 > NAME     COUNT(*) SUM(VALUE) MAX(VALUE) MIN(VALUE) AVG(VALUE)                    COUNT(DISTINCT VALUE)
 > -------- -------- ---------- ---------- ---------- ----------------------------- ---------------------

h2/src/test/org/h2/test/unit/TestFileSystem.java

@@ -56,7 +56,7 @@ public class TestFileSystem extends TestBase {
         stat.execute("backup to '" + baseDir + "/fsJar.zip'");
         conn.close();
 
-        deleteDb(baseDir + "/fsJar");
+        deleteDb("fsJar");
         FileSystem fs = FileSystem.getInstance("zip:" + baseDir + "/fsJar.zip");
         String[] files = fs.listFiles("zip:" + baseDir + "/fsJar.zip");
         for (int i = 0; i < files.length; i++) {