Page store: checksum

h2/src/main/org/h2/index/PageBtreeLeaf.java

@@ -21,6 +21,7 @@ import org.h2.store.PageStore;
  * A b-tree leaf page that contains index data. Format:
  * <ul>
  * <li>page type: byte</li>
+ * <li>checksum: short</li>
  * <li>parent page id (0 for root): int</li>
  * <li>index id: varInt</li>
  * <li>entry count: short</li>
@@ -69,6 +70,7 @@ public class PageBtreeLeaf extends PageBtree {
     private void read() throws SQLException {
         data.reset();
         int type = data.readByte();
+        data.readShortInt();
         this.parentPageId = data.readInt();
         onlyPosition = (type & Page.FLAG_LAST) == 0;
         int indexId = data.readVarInt();
@@ -234,6 +236,7 @@ public class PageBtreeLeaf extends PageBtree {
 
     private void writeHead() {
         data.writeByte((byte) (Page.TYPE_BTREE_LEAF | (onlyPosition ? 0 : Page.FLAG_LAST)));
+        data.writeShortInt(0);
         data.writeInt(parentPageId);
         data.writeVarInt(index.getId());
         data.writeShortInt(entryCount);

h2/src/main/org/h2/index/PageBtreeNode.java

@@ -22,6 +22,7 @@ import org.h2.util.MemoryUtils;
  * A b-tree node page that contains index data. Format:
  * <ul>
  * <li>page type: byte</li>
+ * <li>checksum: short</li>
  * <li>parent page id (0 for root): int</li>
  * <li>index id: varInt</li>
  * <li>count of all children (-1 if not known): int</li>
@@ -82,6 +83,7 @@ public class PageBtreeNode extends PageBtree {
     private void read() throws SQLException {
         data.reset();
         int type = data.readByte();
+        data.readShortInt();
         this.parentPageId = data.readInt();
         onlyPosition = (type & Page.FLAG_LAST) == 0;
         int indexId = data.readVarInt();
@@ -371,6 +373,7 @@ public class PageBtreeNode extends PageBtree {
 
     private void writeHead() {
         data.writeByte((byte) (Page.TYPE_BTREE_NODE | (onlyPosition ? 0 : Page.FLAG_LAST)));
+        data.writeShortInt(0);
         data.writeInt(parentPageId);
         data.writeVarInt(index.getId());
         data.writeInt(rowCount);

h2/src/main/org/h2/index/PageDataLeaf.java

@@ -22,6 +22,7 @@ import org.h2.store.PageStore;
  * A leaf page that contains data of one or multiple rows. Format:
  * <ul>
  * <li>page type: byte</li>
+ * <li>checksum: short</li>
  * <li>parent page id (0 for root): int</li>
  * <li>table id: varInt</li>
  * <li>column count: varInt</li>
@@ -105,6 +106,7 @@ public class PageDataLeaf extends PageData {
     private void read() throws SQLException {
         data.reset();
         int type = data.readByte();
+        data.readShortInt();
         this.parentPageId = data.readInt();
         int tableId = data.readVarInt();
         if (tableId != index.getId()) {
@@ -423,6 +425,7 @@ public class PageDataLeaf extends PageData {
             type = Page.TYPE_DATA_LEAF;
         }
         data.writeByte((byte) type);
+        data.writeShortInt(0);
         data.writeInt(parentPageId);
         data.writeVarInt(index.getId());
         data.writeVarInt(columnCount);

h2/src/main/org/h2/index/PageDataNode.java

@@ -22,6 +22,7 @@ import org.h2.util.MemoryUtils;
  * A leaf page that contains data of one or multiple rows. Format:
  * <ul>
  * <li>page type: byte</li>
+ * <li>checksum: short</li>
  * <li>parent page id (0 for root): int</li>
  * <li>table id: varInt</li>
  * <li>count of all children (-1 if not known): int</li>
@@ -86,6 +87,7 @@ public class PageDataNode extends PageData {
     private void read() throws SQLException {
         data.reset();
         data.readByte();
+        data.readShortInt();
         this.parentPageId = data.readInt();
         int indexId = data.readVarInt();
         if (indexId != index.getId()) {
@@ -313,6 +315,7 @@ public class PageDataNode extends PageData {
 
     private void writeHead() {
         data.writeByte((byte) Page.TYPE_DATA_NODE);
+        data.writeShortInt(0);
         data.writeInt(parentPageId);
         data.writeVarInt(index.getId());
         data.writeInt(rowCountStored);

h2/src/main/org/h2/index/PageDataOverflow.java

@@ -19,10 +19,11 @@ import org.h2.store.PageStore;
  * Overflow data for a leaf page. Format:
  * <ul>
  * <li>page type: byte (0)</li>
- * <li>parent page id (0 for root): int (1-4)</li>
- * <li>more data: next overflow page id: int (5-8)</li>
- * <li>last remaining size: short (5-6)</li>
- * <li>data (9-/7-)</li>
+ * <li>checksum: short (1-2)</li>
+ * <li>parent page id (0 for root): int (3-6)</li>
+ * <li>more data: next overflow page id: int (7-10)</li>
+ * <li>last remaining size: short (7-8)</li>
+ * <li>data (11-/9-)</li>
  * </ul>
  */
 public class PageDataOverflow extends Page {
@@ -30,14 +31,14 @@ public class PageDataOverflow extends Page {
     /**
      * The start of the data in the last overflow page.
      */
-    static final int START_LAST = 7;
+    static final int START_LAST = 9;
 
     /**
      * The start of the data in a overflow page that is not the last one.
      */
-    static final int START_MORE = 9;
+    static final int START_MORE = 11;
 
-    private static final int START_NEXT_OVERFLOW = 5;
+    private static final int START_NEXT_OVERFLOW = 7;
 
     /**
      * The page store.
@@ -130,6 +131,7 @@ public class PageDataOverflow extends Page {
     private void read() throws SQLException {
         data.reset();
         type = data.readByte();
+        data.readShortInt();
         parentPageId = data.readInt();
         if (type == (Page.TYPE_DATA_OVERFLOW | Page.FLAG_LAST)) {
             size = data.readShortInt();
@@ -169,6 +171,7 @@ public class PageDataOverflow extends Page {
 
     private void writeHead() {
         data.writeByte((byte) type);
+        data.writeShortInt(0);
         data.writeInt(parentPageId);
     }
 

h2/src/main/org/h2/store/Page.java

@@ -69,11 +69,6 @@ public abstract class Page extends Record {
      */
     public static final int TYPE_STREAM_DATA = 8;
 
-    /**
-     * A header page.
-     */
-    public static final int TYPE_HEADER = 9;
-
     /**
      * Copy the data to a new location, change the parent to point to the new
      * location, and free up the current page.

h2/src/main/org/h2/store/PageFreeList.java

@@ -13,13 +13,14 @@ import org.h2.util.BitField;
  * The list of free pages of a page store. The format of a free list trunk page
  * is:
  * <ul>
- * <li>page type: byte</li>
- * <li>data (1-)</li>
+ * <li>page type: byte (0)</li>
+ * <li>checksum: short (1-2)</li>
+ * <li>data (3-)</li>
  * </ul>
  */
 public class PageFreeList extends Page {
 
-    private static final int DATA_START = 1;
+    private static final int DATA_START = 3;
 
     private final PageStore store;
     private final BitField used = new BitField();
@@ -140,6 +141,7 @@ public class PageFreeList extends Page {
     private void read() {
         data.reset();
         data.readByte();
+        data.readShortInt();
         for (int i = 0; i < pageCount; i += 8) {
             used.setByte(i, data.readByte() & 255);
         }
@@ -153,6 +155,7 @@ public class PageFreeList extends Page {
     public void write(DataPage buff) throws SQLException {
         data = store.createData();
         data.writeByte((byte) Page.TYPE_FREE_LIST);
+        data.writeShortInt(0);
         for (int i = 0; i < pageCount; i += 8) {
             data.writeByte((byte) used.getByte(i));
         }

h2/src/main/org/h2/store/PageStore.java

@@ -69,19 +69,17 @@ import org.h2.value.ValueString;
  * </ul>
  * The format of page 1 and 2 is:
  * <ul>
- * <li>page type: byte (0)</li>
- * <li>write counter (incremented each time the header changes): long (1-8)</li>
- * <li>log trunk key: int (9-12)</li>
- * <li>log trunk page (0 for none): int (13-16)</li>
- * <li>log data page (0 for none): int (17-20)</li>
- * <li>CRC32 checksum of the page: int (20-23)</li>
+ * <li>CRC32 of the remaining data: int (0-3)</li>
+ * <li>write counter (incremented each time the header changes): long (4-11)</li>
+ * <li>log trunk key: int (12-15)</li>
+ * <li>log trunk page (0 for none): int (16-19)</li>
+ * <li>log data page (0 for none): int (20-23)</li>
  * </ul>
  * Page 3 contains the first free list page.
  * Page 4 contains the meta table root page.
  */
 public class PageStore implements CacheWriter {
 
-    // TODO implement checksum; 0 for empty pages
     // TODO test running out of disk space (using a special file system)
 
     // TODO utf-x: test if it's faster
@@ -455,11 +453,16 @@ public class PageStore implements CacheWriter {
         Data data = createData();
         readPage(pageId, data);
         int type = data.readByte();
+        if (type == Page.TYPE_EMPTY) {
+            return null;
+        }
+        data.readShortInt();
         data.readInt();
+        if (!checksumTest(data.getBytes(), pageId, pageSize)) {
+            throw Message.getSQLException(ErrorCode.FILE_CORRUPTED_1, "wrong checksum");
+        }
         Page p;
         switch (type & ~Page.FLAG_LAST) {
-        case Page.TYPE_EMPTY:
-            return null;
         case Page.TYPE_FREE_LIST:
             p = PageFreeList.read(this, data, pageId);
             break;
@@ -560,26 +563,19 @@ public class PageStore implements CacheWriter {
             }
             page.reset();
             readPage(i, page);
-            int type = page.readByte();
-            if (type == Page.TYPE_HEADER) {
+            CRC32 crc = new CRC32();
+            crc.update(page.getBytes(), 4, pageSize - 4);
+            int expected = (int) crc.getValue();
+            int got = page.readInt();
+            if (expected == got) {
                 writeCount = page.readLong();
                 logKey = page.readInt();
                 logFirstTrunkPage = page.readInt();
                 logFirstDataPage = page.readInt();
-                // read the CRC, then reset it to 0
-                int start = page.length();
-                int got = page.readInt();
-                page.setPos(start);
-                page.writeInt(0);
-                CRC32 crc = new CRC32();
-                crc.update(page.getBytes(), 0, pageSize);
-                int expected = (int) crc.getValue();
-                if (expected == got) {
                 break;
             }
         }
     }
-    }
 
     /**
      * Set the page size. The size must be a power of two. This method must be
@@ -636,14 +632,14 @@ public class PageStore implements CacheWriter {
 
     private void writeVariableHeader() throws SQLException {
         Data page = createData();
-        page.writeByte((byte) Page.TYPE_HEADER);
+        page.writeInt(0);
         page.writeLong(writeCount);
         page.writeInt(logKey);
         page.writeInt(logFirstTrunkPage);
         page.writeInt(logFirstDataPage);
         CRC32 crc = new CRC32();
-        crc.update(page.getBytes(), 0, pageSize);
-        page.writeInt((int) crc.getValue());
+        crc.update(page.getBytes(), 4, pageSize - 4);
+        page.setInt(0, (int) crc.getValue());
         file.seek(pageSize);
         file.write(page.getBytes(), 0, pageSize);
         file.seek(pageSize + pageSize);
@@ -938,9 +934,11 @@ public class PageStore implements CacheWriter {
         if (pageId <= 0) {
             Message.throwInternalError("write to page " + pageId);
         }
+        byte[] bytes = data.getBytes();
+        checksumSet(bytes, pageId);
         synchronized (database) {
             file.seek((long) pageId << pageSizeShift);
-            file.write(data.getBytes(), 0, pageSize);
+            file.write(bytes, 0, pageSize);
             writeCount++;
         }
     }
@@ -1439,32 +1437,41 @@ public class PageStore implements CacheWriter {
         return cache;
     }
 
-    // TODO implement checksum
-//    private void updateChecksum(byte[] d, int pos) {
-//        int ps = pageSize;
-//        int s1 = 255 + (d[0] & 255), s2 = 255 + s1;
-//        s2 += s1 += d[1] & 255;
-//        s2 += s1 += d[(ps >> 1) - 1] & 255;
-//        s2 += s1 += d[ps >> 1] & 255;
-//        s2 += s1 += d[ps - 2] & 255;
-//        s2 += s1 += d[ps - 1] & 255;
-//        d[5] = (byte) (((s1 & 255) + (s1 >> 8)) ^ pos);
-//        d[6] = (byte) (((s2 & 255) + (s2 >> 8)) ^ (pos >> 8));
-//    }
-//
-//    private void verifyChecksum(byte[] d, int pos) throws SQLException {
-//        int ps = pageSize;
-//        int s1 = 255 + (d[0] & 255), s2 = 255 + s1;
-//        s2 += s1 += d[1] & 255;
-//        s2 += s1 += d[(ps >> 1) - 1] & 255;
-//        s2 += s1 += d[ps >> 1] & 255;
-//        s2 += s1 += d[ps - 2] & 255;
-//        s2 += s1 += d[ps - 1] & 255;
-//        if (d[5] != (byte) (((s1 & 255) + (s1 >> 8)) ^ pos)
-//                || d[6] != (byte) (((s2 & 255) + (s2 >> 8)) ^ (pos >> 8))) {
-//            throw Message.getSQLException(
-//                ErrorCode.FILE_CORRUPTED_1, "wrong checksum");
-//        }
-//    }
+    private void checksumSet(byte[] d, int pageId) {
+        int ps = pageSize;
+        int type = d[0];
+        if (type == Page.TYPE_EMPTY) {
+            return;
+        }
+        int s1 = 255 + (type & 255), s2 = 255 + s1;
+        s2 += s1 += d[6] & 255;
+        s2 += s1 += d[(ps >> 1) - 1] & 255;
+        s2 += s1 += d[ps >> 1] & 255;
+        s2 += s1 += d[ps - 2] & 255;
+        s2 += s1 += d[ps - 1] & 255;
+        d[1] = (byte) (((s1 & 255) + (s1 >> 8)) ^ pageId);
+        d[2] = (byte) (((s2 & 255) + (s2 >> 8)) ^ (pageId >> 8));
+    }
+
+    /**
+     * Check if the stored checksum is correct
+     * @param d the data
+     * @param pageId the page id
+     * @return true if it is correct
+     */
+    public static boolean checksumTest(byte[] d, int pageId, int pageSize) {
+        int ps = pageSize;
+        int s1 = 255 + (d[0] & 255), s2 = 255 + s1;
+        s2 += s1 += d[6] & 255;
+        s2 += s1 += d[(ps >> 1) - 1] & 255;
+        s2 += s1 += d[ps >> 1] & 255;
+        s2 += s1 += d[ps - 2] & 255;
+        s2 += s1 += d[ps - 1] & 255;
+        if (d[1] != (byte) (((s1 & 255) + (s1 >> 8)) ^ pageId)
+                || d[2] != (byte) (((s2 & 255) + (s2 >> 8)) ^ (pageId >> 8))) {
+            return false;
+        }
+        return true;
+    }
 
 }

h2/src/main/org/h2/store/PageStreamData.java

@@ -13,14 +13,15 @@ import org.h2.engine.Session;
  * A data page of a stream. The format is:
  * <ul>
  * <li>page type: byte (0)</li>
- * <li>the trunk page id: int (1-4)</li>
- * <li>log key: int (5-8)</li>
- * <li>data (9-)</li>
+ * <li>checksum: short (1-2)</li>
+ * <li>the trunk page id: int (3-6)</li>
+ * <li>log key: int (7-10)</li>
+ * <li>data (11-)</li>
  * </ul>
  */
 public class PageStreamData extends Page {
 
-    private static final int DATA_START = 9;
+    private static final int DATA_START = 11;
 
     private final PageStore store;
     private int trunk;
@@ -69,6 +70,7 @@ public class PageStreamData extends Page {
     private void read() {
         data.reset();
         data.readByte();
+        data.readShortInt();
         trunk = data.readInt();
         logKey = data.readInt();
     }
@@ -83,6 +85,7 @@ public class PageStreamData extends Page {
     void initWrite() {
         data = store.createData();
         data.writeByte((byte) Page.TYPE_STREAM_DATA);
+        data.writeShortInt(0);
         data.writeInt(trunk);
         data.writeInt(logKey);
         remaining = store.getPageSize() - data.length();

h2/src/main/org/h2/store/PageStreamTrunk.java

@@ -14,16 +14,17 @@ import org.h2.engine.Session;
  * page number of the next trunk. The format is:
  * <ul>
  * <li>page type: byte (0)</li>
- * <li>previous trunk page, or 0 if none: int (1-4)</li>
- * <li>log key: int (5-8)</li>
- * <li>next trunk page: int (9-12)</li>
- * <li>number of pages: short (13-14)</li>
- * <li>page ids (15-)</li>
+ * <li>checksum: short (1-2)</li>
+ * <li>previous trunk page, or 0 if none: int (3-6)</li>
+ * <li>log key: int (7-10)</li>
+ * <li>next trunk page: int (11-14)</li>
+ * <li>number of pages: short (15-16)</li>
+ * <li>page ids (17-)</li>
  * </ul>
  */
 public class PageStreamTrunk extends Page {
 
-    private static final int DATA_START = 15;
+    private static final int DATA_START = 17;
 
     private final PageStore store;
     private int parent;
@@ -85,6 +86,7 @@ public class PageStreamTrunk extends Page {
     private void read() {
         data.reset();
         data.readByte();
+        data.readShortInt();
         parent = data.readInt();
         logKey = data.readInt();
         nextTrunk = data.readInt();
@@ -124,6 +126,7 @@ public class PageStreamTrunk extends Page {
     public void write(DataPage buff) throws SQLException {
         data = store.createData();
         data.writeByte((byte) Page.TYPE_STREAM_TRUNK);
+        data.writeShortInt(0);
         data.writeInt(parent);
         data.writeInt(logKey);
         data.writeInt(nextTrunk);

h2/src/main/org/h2/tools/Recover.java

@@ -60,7 +60,6 @@ import org.h2.util.StatementBuilder;
 import org.h2.util.TempFileDeleter;
 import org.h2.util.Tool;
 import org.h2.value.Value;
-import org.h2.value.ValueInt;
 import org.h2.value.ValueLob;
 import org.h2.value.ValueLong;
 
@@ -825,27 +824,22 @@ public class Recover extends Tool implements DataHandler {
                 s.reset();
                 store.seek(i * pageSize);
                 store.readFully(s.getBytes(), 0, pageSize);
-                int type = s.readByte();
+
+                CRC32 crc = new CRC32();
+                crc.update(s.getBytes(), 4, pageSize - 4);
+                int expected = (int) crc.getValue();
+                int got = s.readInt();
                 long writeCounter = s.readLong();
                 int key = s.readInt();
                 int firstTrunkPage = s.readInt();
                 int firstDataPage = s.readInt();
-                int start = s.length();
-                int got = s.readInt();
-                s.setPos(start);
-                s.writeInt(0);
-                CRC32 crc = new CRC32();
-                crc.update(s.getBytes(), 0, pageSize);
-                int expected = (int) crc.getValue();
                 if (expected == got) {
-                    if (logFirstTrunkPage == 0) {
                     logKey = key;
                     logFirstTrunkPage = firstTrunkPage;
                     logFirstDataPage = firstDataPage;
                 }
-                }
                 writer.println("-- head " + i +
-                        ": type: " + type + " writeCounter: " + writeCounter +
+                        ": writeCounter: " + writeCounter +
                         " log key: " + key + " trunk: " + firstTrunkPage + "/" + firstDataPage +
                         " crc expected " + expected +
                         " got " + got + " (" + (expected == got ? "ok" : "different") + ")");
@@ -855,7 +849,7 @@ public class Recover extends Tool implements DataHandler {
 
             s = Data.create(this, pageSize);
             int free = 0;
-            for (long page = 3; page < pageCount; page++) {
+            for (int page = 3; page < pageCount; page++) {
                 s = Data.create(this, pageSize);
                 store.seek(page * pageSize);
                 store.readFully(s.getBytes(), 0, pageSize);
@@ -868,6 +862,10 @@ public class Recover extends Tool implements DataHandler {
                 }
                 boolean last = (type & Page.FLAG_LAST) != 0;
                 type &= ~Page.FLAG_LAST;
+                if (!PageStore.checksumTest(s.getBytes(), page, pageSize)) {
+                    writer.println("-- ERROR: page " + page + " checksum mismatch");
+                }
+                s.readShortInt();
                 switch (type) {
                 // type 1
                 case Page.TYPE_DATA_LEAF: {
@@ -938,7 +936,7 @@ public class Recover extends Tool implements DataHandler {
                     writer.println("-- page " + page + ": log data");
                     break;
                 default:
-                    writer.println("-- page " + page + ": ERROR unknown type " + type);
+                    writer.println("-- ERROR page " + page + " unknown type " + type);
                     break;
                 }
             }
@@ -1240,10 +1238,10 @@ public class Recover extends Tool implements DataHandler {
         for (int i = 0; i < entryCount; i++) {
             int off = offsets[i];
             s.setPos(off);
-            int pos = s.readInt();
+            long key = s.readVarLong();
             Value data;
             if (positionOnly) {
-                data = ValueInt.get(pos);
+                data = ValueLong.get(key);
             } else {
                 try {
                     data = s.readValue();
@@ -1252,7 +1250,7 @@ public class Recover extends Tool implements DataHandler {
                     continue;
                 }
             }
-            writer.println("-- [" + i + "] pos: " + pos + " data: " + data);
+            writer.println("-- [" + i + "] key: " + key + " data: " + data);
         }
     }