Merge pull request #1653 from katzyn/RowList

Store datetime values in local time in temporary storages

h2/src/docsrc/html/changelog.html

@@ -21,6 +21,11 @@ Change Log
 
 <h2>Next Version (unreleased)</h2>
 <ul>
+<li>Issue #1651: TIMESTAMP values near DST may be changed in MVStore database due to UTC-based PageStore format in some
+temporary storages
+</li>
+<li>PR #1650: Fix race in MVStore.close()
+</li>
 <li>Issue #1212: TestDiskFull: The file is locked
 </li>
 <li>PR #1648: Add functions ARRAY_CAT(), ARRAY_APPEND() and ARRAY_SLICE()

h2/src/main/org/h2/command/dml/Update.java

@@ -199,7 +199,6 @@ public class Update extends Prepared {
             // the cached row is already updated - we need the old values
             table.updateRows(this, session, rows);
             if (table.fireRow()) {
-                rows.invalidateCache();
                 for (rows.reset(); rows.hasNext();) {
                     Row o = rows.next();
                     Row n = rows.next();

h2/src/main/org/h2/engine/UndoLog.java

@@ -73,7 +73,7 @@ public class UndoLog {
             long pos = storedEntriesPos.remove(last);
             long end = file.length();
             int bufferLength = (int) (end - pos);
-            Data buff = Data.create(database, bufferLength);
+            Data buff = Data.create(database, bufferLength, true);
             file.seek(pos);
             file.readFully(buff.getBytes(), 0, bufferLength);
             while (buff.length() < bufferLength) {
@@ -146,7 +146,7 @@ public class UndoLog {
                 file.setCheckedWriting(false);
                 file.setLength(FileStore.HEADER_LENGTH);
             }
-            Data buff = Data.create(database, Constants.DEFAULT_PAGE_SIZE);
+            Data buff = Data.create(database, Constants.DEFAULT_PAGE_SIZE, true);
             for (int i = 0; i < records.size(); i++) {
                 UndoLogRecord r = records.get(i);
                 buff.checkCapacity(Constants.DEFAULT_PAGE_SIZE);

h2/src/main/org/h2/result/RowList.java

@@ -32,7 +32,6 @@ public class RowList {
     private final int maxMemory;
     private int memory;
     private boolean written;
-    private boolean readUncached;
 
     /**
      * Construct a new row list for this session.
@@ -91,7 +90,7 @@ public class RowList {
             file = db.openFile(fileName, "rw", false);
             file.setCheckedWriting(false);
             file.seek(FileStore.HEADER_LENGTH);
-            rowBuff = Data.create(db, Constants.DEFAULT_PAGE_SIZE);
+            rowBuff = Data.create(db, Constants.DEFAULT_PAGE_SIZE, true);
             file.seek(FileStore.HEADER_LENGTH);
         }
         Data buff = rowBuff;
@@ -170,9 +169,6 @@ public class RowList {
         int columnCount = buff.readInt();
         long key = buff.readLong();
         int version = buff.readInt();
-        if (readUncached) {
-            key = 0;
-        }
         boolean deleted = buff.readInt() == 1;
         Value[] values = new Value[columnCount];
         for (int i = 0; i < columnCount; i++) {
@@ -243,13 +239,6 @@ public class RowList {
         return size;
     }
 
-    /**
-     * Do not use the cache.
-     */
-    public void invalidateCache() {
-        readUncached = true;
-    }
-
     /**
      * Close the result list and delete the temporary file.
      */

h2/src/main/org/h2/store/Data.java

@@ -114,9 +114,12 @@ public class Data {
      */
     private final DataHandler handler;
 
-    private Data(DataHandler handler, byte[] data) {
+    private final boolean storeLocalTime;
+
+    private Data(DataHandler handler, byte[] data, boolean storeLocalTime) {
         this.handler = handler;
         this.data = data;
+        this.storeLocalTime = storeLocalTime;
     }
 
     /**
@@ -295,10 +298,13 @@ public class Data {
      *
      * @param handler the data handler
      * @param capacity the initial capacity of the buffer
+     * @param storeLocalTime
+     *            store DATE, TIME, and TIMESTAMP values with local time storage
+     *            format
      * @return the buffer
      */
-    public static Data create(DataHandler handler, int capacity) {
-        return new Data(handler, new byte[capacity]);
+    public static Data create(DataHandler handler, int capacity, boolean storeLocalTime) {
+        return new Data(handler, new byte[capacity], storeLocalTime);
     }
 
     /**
@@ -307,10 +313,13 @@ public class Data {
      *
      * @param handler the data handler
      * @param buff the data
+     * @param storeLocalTime
+     *            store DATE, TIME, and TIMESTAMP values with local time storage
+     *            format
      * @return the buffer
      */
-    public static Data create(DataHandler handler, byte[] buff) {
-        return new Data(handler, buff);
+    public static Data create(DataHandler handler, byte[] buff, boolean storeLocalTime) {
+        return new Data(handler, buff, storeLocalTime);
     }
 
     /**
@@ -484,20 +493,48 @@ public class Data {
             break;
         }
         case Value.TIME:
-            writeByte((byte) type);
-            writeVarLong(DateTimeUtils.getTimeLocalWithoutDst(v.getTime()));
+            if (storeLocalTime) {
+                writeByte((byte) LOCAL_TIME);
+                ValueTime t = (ValueTime) v;
+                long nanos = t.getNanos();
+                long millis = nanos / 1_000_000;
+                nanos -= millis * 1_000_000;
+                writeVarLong(millis);
+                writeVarLong(nanos);
+            } else {
+                writeByte((byte) type);
+                writeVarLong(DateTimeUtils.getTimeLocalWithoutDst(v.getTime()));
+            }
             break;
         case Value.DATE: {
-            writeByte((byte) type);
-            long x = DateTimeUtils.getTimeLocalWithoutDst(v.getDate());
-            writeVarLong(x / MILLIS_PER_MINUTE);
+            if (storeLocalTime) {
+                writeByte((byte) LOCAL_DATE);
+                long x = ((ValueDate) v).getDateValue();
+                writeVarLong(x);
+            } else {
+                writeByte((byte) type);
+                long x = DateTimeUtils.getTimeLocalWithoutDst(v.getDate());
+                writeVarLong(x / MILLIS_PER_MINUTE);
+            }
             break;
         }
         case Value.TIMESTAMP: {
-            Timestamp ts = v.getTimestamp();
-            writeByte((byte) type);
-            writeVarLong(DateTimeUtils.getTimeLocalWithoutDst(ts));
-            writeVarInt(ts.getNanos() % 1_000_000);
+            if (storeLocalTime) {
+                writeByte((byte) LOCAL_TIMESTAMP);
+                ValueTimestamp ts = (ValueTimestamp) v;
+                long dateValue = ts.getDateValue();
+                writeVarLong(dateValue);
+                long nanos = ts.getTimeNanos();
+                long millis = nanos / 1_000_000;
+                nanos -= millis * 1_000_000;
+                writeVarLong(millis);
+                writeVarLong(nanos);
+            } else {
+                Timestamp ts = v.getTimestamp();
+                writeByte((byte) type);
+                writeVarLong(DateTimeUtils.getTimeLocalWithoutDst(ts));
+                writeVarInt(ts.getNanos() % 1_000_000);
+            }
             break;
         }
         case Value.TIMESTAMP_TZ: {
@@ -702,8 +739,8 @@ public class Data {
             }
             DbException.throwInternalError("type=" + v.getType());
         }
-        assert pos - start == getValueLen(v, handler)
-                : "value size error: got " + (pos - start) + " expected " + getValueLen(v, handler);
+        assert pos - start == getValueLen(v)
+                : "value size error: got " + (pos - start) + " expected " + getValueLen(v);
     }
 
     /**
@@ -925,17 +962,19 @@ public class Data {
      * @return the number of bytes required to store this value
      */
     public int getValueLen(Value v) {
-        return getValueLen(v, handler);
+        return getValueLen(v, storeLocalTime);
     }
 
     /**
      * Calculate the number of bytes required to encode the given value.
      *
      * @param v the value
-     * @param handler the data handler for lobs
+     * @param storeLocalTime
+     *            calculate size of DATE, TIME, and TIMESTAMP values with local
+     *            time storage format
      * @return the number of bytes required to store this value
      */
-    public static int getValueLen(Value v, DataHandler handler) {
+    public static int getValueLen(Value v, boolean storeLocalTime) {
         if (v == ValueNull.INSTANCE) {
             return 1;
         }
@@ -1020,12 +1059,31 @@ public class Data {
             return 1 + getVarIntLen(scale) + getVarIntLen(bytes.length) + bytes.length;
         }
         case Value.TIME:
+            if (storeLocalTime) {
+                long nanos = ((ValueTime) v).getNanos();
+                long millis = nanos / 1_000_000;
+                nanos -= millis * 1_000_000;
+                return 1 + getVarLongLen(millis) + getVarLongLen(nanos);
+            }
             return 1 + getVarLongLen(DateTimeUtils.getTimeLocalWithoutDst(v.getTime()));
         case Value.DATE: {
+            if (storeLocalTime) {
+                long dateValue = ((ValueDate) v).getDateValue();
+                return 1 + getVarLongLen(dateValue);
+            }
             long x = DateTimeUtils.getTimeLocalWithoutDst(v.getDate());
             return 1 + getVarLongLen(x / MILLIS_PER_MINUTE);
         }
         case Value.TIMESTAMP: {
+            if (storeLocalTime) {
+                ValueTimestamp ts = (ValueTimestamp) v;
+                long dateValue = ts.getDateValue();
+                long nanos = ts.getTimeNanos();
+                long millis = nanos / 1_000_000;
+                nanos -= millis * 1_000_000;
+                return 1 + getVarLongLen(dateValue) + getVarLongLen(millis) +
+                        getVarLongLen(nanos);
+            }
             Timestamp ts = v.getTimestamp();
             return 1 + getVarLongLen(DateTimeUtils.getTimeLocalWithoutDst(ts)) +
                     getVarIntLen(ts.getNanos() % 1_000_000);
@@ -1096,7 +1154,7 @@ public class Data {
             Value[] list = ((ValueCollectionBase) v).getList();
             int len = 1 + getVarIntLen(list.length);
             for (Value x : list) {
-                len += getValueLen(x, handler);
+                len += getValueLen(x, storeLocalTime);
             }
             return len;
         }
@@ -1118,7 +1176,7 @@ public class Data {
                 Value[] row = result.currentRow();
                 for (int i = 0; i < columnCount; i++) {
                     Value val = row[i];
-                    len += getValueLen(val, handler);
+                    len += getValueLen(val, storeLocalTime);
                 }
             }
             len++;
@@ -1360,7 +1418,7 @@ public class Data {
     public static void copyString(Reader source, OutputStream target)
             throws IOException {
         char[] buff = new char[Constants.IO_BUFFER_SIZE];
-        Data d = new Data(null, new byte[3 * Constants.IO_BUFFER_SIZE]);
+        Data d = new Data(null, new byte[3 * Constants.IO_BUFFER_SIZE], false);
         while (true) {
             int l = source.read(buff);
             if (l < 0) {

h2/src/main/org/h2/store/FileStoreInputStream.java

@@ -33,7 +33,7 @@ public class FileStoreInputStream extends InputStream {
         } else {
             compress = null;
         }
-        page = Data.create(handler, Constants.FILE_BLOCK_SIZE);
+        page = Data.create(handler, Constants.FILE_BLOCK_SIZE, true);
         try {
             if (store.length() <= FileStore.HEADER_LENGTH) {
                 close();

h2/src/main/org/h2/store/FileStoreOutputStream.java

@@ -31,7 +31,7 @@ public class FileStoreOutputStream extends OutputStream {
             this.compress = null;
             this.compressionAlgorithm = null;
         }
-        page = Data.create(handler, Constants.FILE_BLOCK_SIZE);
+        page = Data.create(handler, Constants.FILE_BLOCK_SIZE, true);
     }
 
     @Override

h2/src/main/org/h2/store/PageStore.java

@@ -869,7 +869,7 @@ public class PageStore implements CacheWriter {
     private void readStaticHeader() {
         file.seek(FileStore.HEADER_LENGTH);
         Data page = Data.create(database,
-                new byte[PAGE_SIZE_MIN - FileStore.HEADER_LENGTH]);
+                new byte[PAGE_SIZE_MIN - FileStore.HEADER_LENGTH], false);
         file.readFully(page.getBytes(), 0,
                 PAGE_SIZE_MIN - FileStore.HEADER_LENGTH);
         readCount++;
@@ -941,7 +941,7 @@ public class PageStore implements CacheWriter {
     }
 
     private void writeStaticHeader() {
-        Data page = Data.create(database, new byte[pageSize - FileStore.HEADER_LENGTH]);
+        Data page = Data.create(database, new byte[pageSize - FileStore.HEADER_LENGTH], false);
         page.writeInt(pageSize);
         page.writeByte((byte) WRITE_VERSION);
         page.writeByte((byte) READ_VERSION);
@@ -1281,7 +1281,7 @@ public class PageStore implements CacheWriter {
      * @return the data page.
      */
     public Data createData() {
-        return Data.create(database, new byte[pageSize]);
+        return Data.create(database, new byte[pageSize], false);
     }
 
     /**

h2/src/main/org/h2/tools/Recover.java

@@ -486,7 +486,7 @@ public class Recover extends Tool implements DataHandler {
             } catch (Exception e) {
                 writeError(writer, e);
             }
-            Data s = Data.create(this, 128);
+            Data s = Data.create(this, 128, false);
             seek(0);
             store.readFully(s.getBytes(), 0, 128);
             s.setPos(48);
@@ -503,7 +503,7 @@ public class Recover extends Tool implements DataHandler {
             }
             long pageCount = length / pageSize;
             parents = new int[(int) pageCount];
-            s = Data.create(this, pageSize);
+            s = Data.create(this, pageSize, false);
             for (long i = 3; i < pageCount; i++) {
                 s.reset();
                 seek(i);
@@ -513,7 +513,7 @@ public class Recover extends Tool implements DataHandler {
                 parents[(int) i] = s.readInt();
             }
             int logKey = 0, logFirstTrunkPage = 0, logFirstDataPage = 0;
-            s = Data.create(this, pageSize);
+            s = Data.create(this, pageSize, false);
             for (long i = 1;; i++) {
                 if (i == 3) {
                     break;
@@ -789,9 +789,9 @@ public class Recover extends Tool implements DataHandler {
     }
 
     private void dumpPageStore(PrintWriter writer, long pageCount) {
-        Data s = Data.create(this, pageSize);
+        Data s = Data.create(this, pageSize, false);
         for (long page = 3; page < pageCount; page++) {
-            s = Data.create(this, pageSize);
+            s = Data.create(this, pageSize, false);
             seek(page);
             store.readFully(s.getBytes(), 0, pageSize);
             dumpPage(writer, s, page, pageCount);
@@ -899,7 +899,7 @@ public class Recover extends Tool implements DataHandler {
     private void dumpPageLogStream(PrintWriter writer, int logKey,
             int logFirstTrunkPage, int logFirstDataPage, long pageCount)
             throws IOException {
-        Data s = Data.create(this, pageSize);
+        Data s = Data.create(this, pageSize, false);
         DataReader in = new DataReader(
                 new PageInputStream(writer, this, store, logKey,
                 logFirstTrunkPage, logFirstDataPage, pageSize)
@@ -968,7 +968,7 @@ public class Recover extends Tool implements DataHandler {
                 }
                 writer.println("-- undo page " + pageId + " " + typeName);
                 if (trace) {
-                    Data d = Data.create(null, data);
+                    Data d = Data.create(null, data, false);
                     dumpPage(writer, d, pageId, pageCount);
                 }
             } else if (x == PageLog.ADD) {
@@ -1094,7 +1094,7 @@ public class Recover extends Tool implements DataHandler {
             this.logKey = logKey - 1;
             this.nextTrunkPage = firstTrunkPage;
             this.dataPage = firstDataPage;
-            page = Data.create(handler, pageSize);
+            page = Data.create(handler, pageSize, false);
         }
 
         @Override
@@ -1379,7 +1379,7 @@ public class Recover extends Tool implements DataHandler {
             writer.println("--   empty: " + empty);
         }
         if (!last) {
-            Data s2 = Data.create(this, pageSize);
+            Data s2 = Data.create(this, pageSize, false);
             s.setPos(pageSize);
             long parent = pageId;
             while (true) {

h2/src/test/org/h2/test/unit/TestDataPage.java

@@ -72,7 +72,7 @@ public class TestDataPage extends TestBase implements DataHandler {
     }
 
     private static void testPerformance() {
-        Data data = Data.create(null, 1024);
+        Data data = Data.create(null, 1024, false);
         for (int j = 0; j < 4; j++) {
             long time = System.nanoTime();
             for (int i = 0; i < 100000; i++) {
@@ -217,7 +217,17 @@ public class TestDataPage extends TestBase implements DataHandler {
     }
 
     private void testValue(Value v) {
-        Data data = Data.create(null, 1024);
+        testValue(v, false);
+        switch (v.getType()) {
+        case Value.DATE:
+        case Value.TIME:
+        case Value.TIMESTAMP:
+            testValue(v, true);
+        }
+    }
+
+    private void testValue(Value v, boolean storeLocalTime) {
+        Data data = Data.create(null, 1024, storeLocalTime);
         data.checkCapacity((int) v.getPrecision());
         data.writeValue(v);
         data.writeInt(123);
@@ -229,7 +239,7 @@ public class TestDataPage extends TestBase implements DataHandler {
     }
 
     private void testAll() {
-        Data page = Data.create(this, 128);
+        Data page = Data.create(this, 128, false);
 
         char[] data = new char[0x10000];
         for (int i = 0; i < data.length; i++) {

h2/src/test/org/h2/test/utils/MemoryFootprint.java

@@ -33,7 +33,7 @@ public class MemoryFootprint {
         print("BigDecimal", new BigDecimal("0"));
         print("BigInteger", new BigInteger("0"));
         print("String", new String("Hello"));
-        print("Data", Data.create(null, 10));
+        print("Data", Data.create(null, 10, false));
         print("Row", new RowImpl(new Value[0], 0));
         System.out.println();
         for (int i = 1; i < 128; i += i) {