Sharded map (work in progress)

h2/src/test/org/h2/test/store/TestShardedMap.java

+/*
+ * Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.test.store;
+
+import java.util.TreeMap;
+
+import org.h2.dev.cluster.ShardedMap;
+import org.h2.test.TestBase;
+
+/**
+ * Test sharded maps.
+ */
+public class TestShardedMap extends TestBase {
+
+    /**
+     * Run just this test.
+     *
+     * @param a ignored
+     */
+    public static void main(String... a) throws Exception {
+        TestBase.createCaller().init().test();
+    }
+
+    @Override
+    public void test() {
+        testLinearSplit();
+        testReplication();
+        testOverlap();
+    }
+
+    private void testLinearSplit() {
+        ShardedMap<Integer, Integer> map = new ShardedMap<Integer, Integer>();
+        TreeMap<Integer, Integer> a = new TreeMap<Integer, Integer>();
+        TreeMap<Integer, Integer> b = new TreeMap<Integer, Integer>();
+        map.addMap(a, null, 5);
+        map.addMap(b, 5, null);
+        for (int i = 0; i < 10; i++) {
+            map.put(i, i * 10);
+        }
+        assertEquals(10, map.size());
+        for (int i = 0; i < 10; i++) {
+            assertEquals(i * 10, map.get(i).intValue());
+        }        
+        assertEquals("[0, 1, 2, 3, 4]",
+                a.keySet().toString());
+        assertEquals("[5, 6, 7, 8, 9]",
+                b.keySet().toString());
+        assertEquals("[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]",
+                map.keySet().toString());
+        assertEquals(10, map.sizeAsLong());
+    }
+    
+    private void testReplication() {
+        ShardedMap<Integer, Integer> map = new ShardedMap<Integer, Integer>();
+        TreeMap<Integer, Integer> a = new TreeMap<Integer, Integer>();
+        TreeMap<Integer, Integer> b = new TreeMap<Integer, Integer>();
+        map.addMap(a, null, null);
+        map.addMap(b, null, null);
+        for (int i = 0; i < 10; i++) {
+            map.put(i, i * 10);
+        }
+        assertEquals(10, map.size());
+        for (int i = 0; i < 10; i++) {
+            assertEquals(i * 10, map.get(i).intValue());
+        }        
+        assertEquals("[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]",
+                a.keySet().toString());
+        assertEquals("[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]",
+                b.keySet().toString());
+        assertEquals("[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]",
+                map.keySet().toString());
+        assertEquals(10, map.sizeAsLong());
+    }
+
+    private void testOverlap() {
+        ShardedMap<Integer, Integer> map = new ShardedMap<Integer, Integer>();
+        TreeMap<Integer, Integer> a = new TreeMap<Integer, Integer>();
+        TreeMap<Integer, Integer> b = new TreeMap<Integer, Integer>();
+        map.addMap(a, null, 10);
+        map.addMap(b, 5, null);
+        for (int i = 0; i < 20; i++) {
+            map.put(i, i * 10);
+        }
+        // overlap: size is unknown
+        assertEquals(-1, map.size());
+        for (int i = 0; i < 20; i++) {
+            assertEquals(i * 10, map.get(i).intValue());
+        }        
+        assertEquals("[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]",
+                a.keySet().toString());
+        assertEquals("[5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]",
+                b.keySet().toString());
+        assertEquals(-1, map.sizeAsLong());
+    }
+
+}
\ No newline at end of file

h2/src/tools/org/h2/dev/cluster/ShardedMap.java

+/*
+ * Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License,
+ * Version 1.0, and under the Eclipse Public License, Version 1.0
+ * (http://h2database.com/html/license.html).
+ * Initial Developer: H2 Group
+ */
+package org.h2.dev.cluster;
+
+import java.util.AbstractMap;
+import java.util.AbstractSet;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+import org.h2.mvstore.DataUtils;
+import org.h2.mvstore.type.DataType;
+import org.h2.mvstore.type.ObjectDataType;
+
+/**
+ * A sharded map. It is typically split into multiple sub-maps that don't have
+ * overlapping keys.
+ * 
+ * @param <K> the key type
+ * @param <V> the value type
+ */
+public class ShardedMap<K, V> extends AbstractMap<K, V>
+        implements Map<K, V> {
+
+    private final DataType keyType;
+
+    /**
+     * The shards. Each shard has a minimum and a maximum key (null for no
+     * limit). Key ranges are ascending but can overlap, in which case entries
+     * may be stored in multiple maps. If that is the case, for read operations,
+     * the entry in the first map is used, and for write operations, the changes
+     * are applied to all maps within the range.
+     */
+    private Shard<K, V>[] shards;
+    
+    public ShardedMap() {
+        this(new ObjectDataType());
+    }
+
+    @SuppressWarnings("unchecked")
+    public ShardedMap(DataType keyType) {
+        this.keyType = keyType;
+        shards = new Shard[0];
+    }
+    
+    public static long getSize(Map<?, ?> map) {
+        if (map instanceof LargeMap) {
+            return ((LargeMap) map).sizeAsLong();
+        }
+        return map.size();
+    }
+    
+    /**
+     * Add the given shard.
+     *
+     * @param map the map
+     * @param min the lowest key, or null if no limit
+     * @param max the highest key, or null if no limit
+     */
+    public void addMap(Map<K, V> map, K min, K max) {
+        if (min != null && max != null && keyType.compare(min, max) > 0) {
+            DataUtils.newIllegalArgumentException("Invalid range: {0} .. {1}", min, max);
+        }
+        int len = shards.length + 1;
+        Shard<K, V>[] newShards = Arrays.copyOf(shards, len);
+        Shard<K, V> newShard = new Shard<K, V>();
+        newShard.map = map;
+        newShard.minIncluding = min;
+        newShard.maxExcluding = max;
+        newShards[len - 1] = newShard;
+        shards = newShards;
+    }
+    
+    private boolean isInRange(K key, Shard<K, V> shard) {
+        if (shard.minIncluding != null) {
+            if (keyType.compare(key, shard.minIncluding) < 0) {
+                return false;
+            }
+        }
+        if (shard.maxExcluding != null) {
+            if (keyType.compare(key, shard.maxExcluding) >= 0) {
+                return false;
+            }
+        }
+        return true;
+    }
+    
+    @Override
+    public int size() {
+        long size = sizeAsLong();
+        return size > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) size;
+    }
+    
+    public long sizeAsLong() {
+        Shard<K, V>[] copy = shards;
+        for (Shard<K, V> s : copy) {
+            if (s.minIncluding == null && s.maxExcluding == null) {
+                return getSize(s.map);
+            }
+        }
+        if (isSimpleSplit(copy)) {
+            long size = 0;
+            for (Shard<K, V> s : copy) {
+                size += getSize(s.map);
+            }
+            return size;
+        }
+        return -1;
+    }
+    
+    private boolean isSimpleSplit(Shard<K, V>[] shards) {
+        K last = null;
+        for (int i = 0; i < shards.length; i++) {
+            Shard<K, V> s = shards[i];
+            if (last == null) {
+                if (s.minIncluding != null) {
+                    return false;
+                }
+            } else if (keyType.compare(last, s.minIncluding) != 0) {
+                return false;
+            }
+            if (s.maxExcluding == null) {
+                return i == shards.length - 1;
+            } 
+            last = s.maxExcluding;
+        }
+        return last == null;
+    }
+
+    @Override
+    public V put(K key, V value) {
+        V result = null;
+        Shard<K, V>[] copy = shards;
+        for (Shard<K, V> s : copy) {
+            if (isInRange(key, s)) {
+                V r = s.map.put(key, value);
+                if (result == null) {
+                    result = r;
+                }
+            }
+        }
+        return result;
+    }
+
+    @Override
+    public V get(Object key) {
+        @SuppressWarnings("unchecked")
+        K k = (K) key;
+        Shard<K, V>[] copy = shards;
+        for (Shard<K, V> s : copy) {
+            if (isInRange(k, s)) {
+                return s.map.get(k);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Set<Entry<K, V>> entrySet() {
+        Shard<K, V>[] copy = shards;
+        for (Shard<K, V> s : copy) {
+            if (s.minIncluding == null && s.maxExcluding == null) {
+                return s.map.entrySet();
+            }
+        }
+        if (isSimpleSplit(copy)) {
+            return new CombinedSet<K, V>(size(), copy);
+        }
+        return null;
+    }
+
+    /**
+     * A subset of a map.
+     *
+     * @param <K> the key type
+     * @param <V> the value type
+     */
+    static class Shard<K, V> {
+
+        /**
+         * The lowest key, or null if no limit.
+         */
+        K minIncluding;
+
+        /**
+         * A key higher than the highest key, or null if no limit.
+         */
+        K maxExcluding;
+
+        /**
+         * The backing map.
+         */
+        Map<K, V> map;
+
+        @Override
+        public String toString() {
+            StringBuilder buff = new StringBuilder();
+            if (minIncluding != null) {
+                buff.append('[').append(minIncluding);
+            }
+            buff.append("..");
+            if (maxExcluding != null) {
+                buff.append(maxExcluding).append(')');
+            }
+            return buff.toString();
+        }
+
+    }
+    
+    /**
+     * A combination of multiple sets.
+     *
+     * @param <K> the key type
+     * @param <V> the value type
+     */
+    private static class CombinedSet<K, V> extends AbstractSet<Entry<K, V>> {
+        
+        final int size;
+        final Shard<K, V>[] shards;
+        
+        CombinedSet(int size, Shard<K, V>[] shards) {
+            this.size = size;
+            this.shards = shards;
+        }
+
+        @Override
+        public Iterator<Entry<K, V>> iterator() {
+            return new Iterator<Entry<K, V>>() {
+                
+                boolean init;
+                Entry<K, V> current;
+                Iterator<Entry<K, V>> currentIterator;
+                int shardIndex;
+                
+                private void fetchNext() {
+                    while (currentIterator == null || !currentIterator.hasNext()) {
+                        if (shardIndex >= shards.length) {
+                            current = null;
+                            return;
+                        }
+                        currentIterator = shards[shardIndex++].map.entrySet().iterator();
+                    }
+                    current = currentIterator.next();
+                }
+
+                @Override
+                public boolean hasNext() {
+                    if (!init) {
+                        fetchNext();
+                        init = true;
+                    }
+                    return current != null;
+                }
+
+                @Override
+                public Entry<K, V> next() {
+                    if (!hasNext()) {
+                        throw new NoSuchElementException();
+                    }
+                    Entry<K, V> e = current;
+                    fetchNext();
+                    return e;
+                }
+
+                @Override
+                public void remove() {
+                    throw new UnsupportedOperationException();
+                }
+                
+            };
+        }
+
+        @Override
+        public int size() {
+            return size;
+        }
+        
+    }
+    
+    /**
+     * A large map.
+     */
+    public interface LargeMap {
+        long sizeAsLong();
+    }
+
+    /**
+     * A map that can efficiently return the index of a key, and the key at a
+     * given index.
+     */
+    public interface CountedMap<K, V> {
+        K getKey(long index);
+        long getKeyIndex(K key);
+    }
+
+}

h2/src/tools/org/h2/dev/cluster/package.html

+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<!--
+Copyright 2004-2013 H2 Group. Multiple-Licensed under the H2 License, Version 1.0,
+and under the Eclipse Public License, Version 1.0
+(http://h2database.com/html/license.html).
+Initial Developer: H2 Group
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head><meta http-equiv="Content-Type" content="text/html;charset=utf-8" /><title>
+Javadoc package documentation
+</title></head><body style="font: 9pt/130% Tahoma, Arial, Helvetica, sans-serif; font-weight: normal;"><p>
+
+A clustering implementation.
+
+</p></body></html>
\ No newline at end of file