You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2017/07/04 10:02:08 UTC

[02/50] ignite git commit: Storing binary metadata in FS

Storing binary metadata in FS


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/92f66b23
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/92f66b23
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/92f66b23

Branch: refs/heads/master
Commit: 92f66b23e4beb613b7dead87111ccc54aef8bf22
Parents: 94fd731
Author: Sergey Chugunov <se...@gmail.com>
Authored: Fri Jun 23 11:24:52 2017 +0300
Committer: Sergey Chugunov <se...@gmail.com>
Committed: Fri Jun 23 11:24:52 2017 +0300

----------------------------------------------------------------------
 .../discovery/GridDiscoveryManager.java         |   2 +-
 .../cache/binary/BinaryMetadataFileStore.java   | 114 ++++
 .../cache/binary/BinaryMetadataTransport.java   |  15 +-
 .../binary/CacheObjectBinaryProcessorImpl.java  |  14 +-
 ...tePdsBinaryMetadataOnClusterRestartTest.java | 514 +++++++++++++++++++
 .../testframework/junits/GridAbstractTest.java  |   1 +
 .../IgnitePdsWithIndexingCoreTestSuite.java     |   3 +
 7 files changed, 659 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/92f66b23/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index de8daa7..c38e37a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -1947,7 +1947,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
                 inject();
             }
             catch (IgniteCheckedException e) {
-                throw new IgniteException("Failed to init consisten ID.", e);
+                throw new IgniteException("Failed to init consistent ID.", e);
             }
 
             consistentId = getSpi().consistentId();

http://git-wip-us.apache.org/repos/asf/ignite/blob/92f66b23/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
new file mode 100644
index 0000000..e682593
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.binary;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.binary.BinaryMetadata;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Class handles saving/restoring binary metadata to/from disk.
+ *
+ * Current implementation needs to be rewritten as it issues IO operations from discovery thread
+ * which may lead to segmentation of nodes from cluster.
+ */
+class BinaryMetadataFileStore {
+    /** */
+    private File workDir;
+
+    /** */
+    private final ConcurrentMap<Integer, BinaryMetadataHolder> metadataLocCache;
+
+    /** */
+    private final GridKernalContext ctx;
+
+    /** */
+    private final IgniteLogger log;
+
+    /**
+     * @param metadataLocCache Metadata locale cache.
+     * @param ctx Context.
+     * @param log Logger.
+     */
+    BinaryMetadataFileStore(ConcurrentMap<Integer, BinaryMetadataHolder> metadataLocCache, GridKernalContext ctx, IgniteLogger log) throws IgniteCheckedException {
+        this.metadataLocCache = metadataLocCache;
+        this.ctx = ctx;
+        this.log = log;
+
+        if (!ctx.config().isPersistentStoreEnabled())
+            return;
+
+        String consId = U.maskForFileName(ctx.discovery().consistentId().toString());
+
+        workDir = new File(U.resolveWorkDirectory(
+            ctx.config().getWorkDirectory(),
+            "binary_meta",
+            false
+        ),
+            consId);
+
+        U.ensureDirectory(workDir, "directory for serialized binary metadata", log);
+    }
+
+    /**
+     * @param binMeta Binary metadata to be written to disk.
+     */
+    void saveMetadata(BinaryMetadata binMeta) {
+        if (!ctx.config().isPersistentStoreEnabled())
+            return;
+
+        try {
+            File file = new File(workDir, Integer.toString(binMeta.typeId()) + ".bin");
+
+            try(FileOutputStream out = new FileOutputStream(file, false)) {
+                byte[] marshalled = U.marshal(ctx, binMeta);
+
+                out.write(marshalled);
+            }
+        }
+        catch (Exception e) {
+            U.warn(log, "Failed to save metadata for typeId: " + binMeta.typeId() +
+                "; exception was thrown: " + e.getMessage());
+        }
+    }
+
+    /**
+     * Restores metadata on startup of {@link CacheObjectBinaryProcessorImpl} but before starting discovery.
+     */
+    void restoreMetadata() {
+        if (!ctx.config().isPersistentStoreEnabled())
+            return;
+
+        for (File file : workDir.listFiles()) {
+            try (FileInputStream in = new FileInputStream(file)) {
+                BinaryMetadata meta = U.unmarshal(ctx.config().getMarshaller(), in, U.resolveClassLoader(ctx.config()));
+
+                metadataLocCache.put(meta.typeId(), new BinaryMetadataHolder(meta, 0, 0));
+            }
+            catch (Exception e) {
+                U.warn(log, "Failed to restore metadata from file: " + file.getName() +
+                    "; exception was thrown: " + e.getMessage());
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/92f66b23/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
index e4df075..59c4856 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
@@ -75,6 +75,9 @@ final class BinaryMetadataTransport {
     private final ConcurrentMap<Integer, BinaryMetadataHolder> metaLocCache;
 
     /** */
+    private final BinaryMetadataFileStore metadataFileStore;
+
+    /** */
     private final Queue<MetadataUpdateResultFuture> unlabeledFutures = new ConcurrentLinkedQueue<>();
 
     /** */
@@ -91,12 +94,20 @@ final class BinaryMetadataTransport {
 
     /**
      * @param metaLocCache Metadata locale cache.
+     * @param metadataFileStore File store for binary metadata.
      * @param ctx Context.
      * @param log Logger.
      */
-    BinaryMetadataTransport(ConcurrentMap<Integer, BinaryMetadataHolder> metaLocCache, final GridKernalContext ctx, IgniteLogger log) {
+    BinaryMetadataTransport(
+        ConcurrentMap<Integer, BinaryMetadataHolder> metaLocCache,
+        BinaryMetadataFileStore metadataFileStore,
+        final GridKernalContext ctx,
+        IgniteLogger log
+    ) {
         this.metaLocCache = metaLocCache;
 
+        this.metadataFileStore = metadataFileStore;
+
         this.ctx = ctx;
 
         this.log = log;
@@ -441,6 +452,8 @@ final class BinaryMetadataTransport {
                 }
 
                 metaLocCache.put(typeId, new BinaryMetadataHolder(holder.metadata(), holder.pendingVersion(), newAcceptedVer));
+
+                metadataFileStore.saveMetadata(holder.metadata());
             }
 
             for (BinaryMetadataUpdatedListener lsnr : binaryUpdatedLsnrs)

http://git-wip-us.apache.org/repos/asf/ignite/blob/92f66b23/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 83e2997..329c50e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -107,6 +107,9 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
     private GridBinaryMarshaller binaryMarsh;
 
     /** */
+    private BinaryMetadataFileStore metadataFileStore;
+
+    /** */
     @GridToStringExclude
     private IgniteBinary binaries;
 
@@ -143,7 +146,11 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
             if (ctx.clientNode())
                 ctx.event().addLocalEventListener(clientDisconLsnr, EVT_CLIENT_NODE_DISCONNECTED);
 
-            transport = new BinaryMetadataTransport(metadataLocCache, ctx, log);
+            metadataFileStore = new BinaryMetadataFileStore(metadataLocCache, ctx, log);
+
+            metadataFileStore.restoreMetadata();
+
+            transport = new BinaryMetadataTransport(metadataLocCache, metadataFileStore, ctx, log);
 
             BinaryMetadataHandler metaHnd = new BinaryMetadataHandler() {
                 @Override public void addMeta(int typeId, BinaryType newMeta) throws BinaryObjectException {
@@ -158,7 +165,7 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
                         BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, ((BinaryTypeImpl)newMeta).metadata());
 
                         if (oldMeta != mergedMeta)
-                            metadataLocCache.putIfAbsent(typeId, new BinaryMetadataHolder(mergedMeta, 0, 0));
+                            metadataLocCache.put(typeId, new BinaryMetadataHolder(mergedMeta, 0, 0));
 
                         return;
                     }
@@ -881,6 +888,9 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
                     log.debug("Received metadata on join: " + localHolder);
 
                 metadataLocCache.put(e.getKey(), localHolder);
+
+                if (!ctx.clientNode())
+                    metadataFileStore.saveMetadata(holder.metadata());
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/92f66b23/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
new file mode 100644
index 0000000..cc3820b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
@@ -0,0 +1,514 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryType;
+import org.apache.ignite.binary.BinaryTypeConfiguration;
+import org.apache.ignite.cache.CacheKeyConfiguration;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_NAME = "cache1";
+
+    /** */
+    private static final String DYNAMIC_TYPE_NAME = "DynamicType";
+
+    /** */
+    private static final String DYNAMIC_INT_FIELD_NAME = "intField";
+
+    /** */
+    private static final String DYNAMIC_STR_FIELD_NAME = "strField";
+
+    /** */
+    private boolean clientMode;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(clientMode);
+
+        cfg.setPersistentStoreConfiguration(
+            new PersistentStoreConfiguration()
+                .setWalMode(WALMode.LOG_ONLY)
+        );
+
+        BinaryConfiguration bCfg = new BinaryConfiguration();
+
+        BinaryTypeConfiguration binaryEnumCfg = new BinaryTypeConfiguration(EnumType.class.getName());
+
+        binaryEnumCfg.setEnum(true);
+        binaryEnumCfg.setEnumValues(F.asMap(EnumType.ENUM_VAL_0.name(),
+            EnumType.ENUM_VAL_0.ordinal(),
+            EnumType.ENUM_VAL_1.name(),
+            EnumType.ENUM_VAL_1.ordinal()));
+
+        bCfg.setTypeConfigurations(Arrays.asList(binaryEnumCfg));
+
+        cfg.setBinaryConfiguration(bCfg);
+
+        CacheKeyConfiguration dynamicMetaKeyCfg = new CacheKeyConfiguration(DYNAMIC_TYPE_NAME, DYNAMIC_INT_FIELD_NAME);
+
+        cfg.setCacheKeyConfiguration(dynamicMetaKeyCfg);
+
+        cfg.setCacheConfiguration(new CacheConfiguration()
+            .setName(CACHE_NAME)
+            .setAffinity(new RendezvousAffinityFunction())
+            .setCacheMode(CacheMode.REPLICATED));
+
+        return cfg;
+    }
+
+    /**
+     * Test verifies that binary metadata from regular java classes is saved and restored correctly
+     * on cluster restart.
+     */
+    public void testStaticMetadataIsRestoredOnRestart() throws Exception {
+        clientMode = false;
+
+        startGrids(2);
+
+        Ignite ignite0 = grid(0);
+
+        ignite0.active(true);
+
+        IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
+
+        cache0.put(0, new TestValue1(0));
+        cache0.put(1, new TestValue2("value"));
+
+        stopAllGrids();
+
+        startGrids(2);
+
+        ignite0 = grid(0);
+
+        ignite0.active(true);
+
+        examineStaticMetadata(2);
+
+        startGrid(2);
+
+        startGrid(3);
+
+        awaitPartitionMapExchange();
+
+        examineStaticMetadata(4);
+    }
+
+    /**
+     * @param nodesCnt Number of nodes in grid.
+     */
+    private void examineStaticMetadata(int nodesCnt) {
+        for (int i = 0; i < nodesCnt; i++) {
+            IgniteCache cache = grid(i).cache(CACHE_NAME).withKeepBinary();
+
+            BinaryObject o1 = (BinaryObject) cache.get(0);
+
+            TestValue1 t1 = o1.deserialize();
+
+            assertEquals(0, t1.getValue());
+
+            BinaryObject o2 = (BinaryObject) cache.get(1);
+
+            TestValue2 t2 = o2.deserialize();
+
+            assertEquals("value", t2.getValue());
+
+            assertEquals(TestValue1.class.getName(), o1.type().typeName());
+            assertEquals(TestValue2.class.getName(), o2.type().typeName());
+
+            assertEquals("val", o1.type().affinityKeyFieldName());
+        }
+    }
+
+    /**
+     * @param nodesCount Nodes count.
+     */
+    private void examineDynamicMetadata(int nodesCount, BinaryObjectExaminer... examiners) {
+        for (int i = 0; i < nodesCount; i++) {
+            Ignite ignite = grid(i);
+
+            for (BinaryObjectExaminer examiner : examiners)
+                examiner.examine(ignite.cache(CACHE_NAME).withKeepBinary());
+        }
+    }
+
+    /**
+     * Test verifies that metadata for binary types built with BinaryObjectBuilder is saved and updated correctly
+     * on cluster restart.
+     */
+    public void testDynamicMetadataIsRestoredOnRestart() throws Exception {
+        clientMode = false;
+        //1: start two nodes, add single BinaryObject
+        startGrids(2);
+
+        Ignite ignite0 = grid(0);
+
+        ignite0.active(true);
+
+        IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
+
+        BinaryObject bo = ignite0
+            .binary()
+            .builder(DYNAMIC_TYPE_NAME)
+            .setField(DYNAMIC_INT_FIELD_NAME, 10)
+            .build();
+
+        cache0.put(2, bo);
+
+        stopAllGrids();
+
+        //2: start two nodes, check BinaryObject added on step 1) (field values, metadata info),
+        // modify BinaryObject type, add new instance of modified type
+        startGrids(2);
+
+        ignite0 = grid(0);
+
+        ignite0.active(true);
+
+        examineDynamicMetadata(2, contentExaminer0, structureExaminer0);
+
+        Ignite ignite1 = grid(1);
+
+        BinaryObject bo1 = ignite1
+            .binary()
+            .builder(DYNAMIC_TYPE_NAME)
+            .setField(DYNAMIC_INT_FIELD_NAME, 20)
+            .setField(DYNAMIC_STR_FIELD_NAME, "str")
+            .build();
+
+        ignite1.cache(CACHE_NAME).put(3, bo1);
+
+        stopAllGrids();
+
+        //3: start two nodes, check both BinaryObject instances,
+        // start two additional nodes, check BO instances on all nodes
+        startGrids(2);
+
+        ignite0 = grid(0);
+
+        ignite0.active(true);
+
+        examineDynamicMetadata(2, contentExaminer0, contentExaminer1, structureExaminer1);
+
+        startGrid(2);
+
+        startGrid(3);
+
+        awaitPartitionMapExchange();
+
+        examineDynamicMetadata(4, contentExaminer0, contentExaminer1, structureExaminer1);
+    }
+
+    /**
+     *
+     */
+    public void testBinaryEnumMetadataIsRestoredOnRestart() throws Exception {
+        clientMode = false;
+
+        Ignite ignite0 = startGrids(2);
+
+        ignite0.active(true);
+
+        BinaryObject enumBo0 = ignite0.binary().buildEnum(EnumType.class.getName(), 0);
+
+        ignite0.cache(CACHE_NAME).put(4, enumBo0);
+
+        stopAllGrids();
+
+        ignite0 = startGrids(2);
+
+        ignite0.active(true);
+
+        startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        examineDynamicMetadata(3, enumExaminer0);
+
+        stopAllGrids();
+
+        ignite0 = startGrids(3);
+
+        ignite0.active(true);
+
+        clientMode = true;
+
+        startGrid(3);
+
+        awaitPartitionMapExchange();
+
+        examineDynamicMetadata(4, enumExaminer0);
+    }
+
+    /**
+     * Test verifies that metadata is saved, stored and delivered to client nodes correctly.
+     */
+    public void testMixedMetadataIsRestoredOnRestart() throws Exception {
+        clientMode = false;
+
+        //1: starts 4 nodes one by one and adds java classes and classless BinaryObjects
+        // to the same replicated cache from different nodes.
+        // Examines all objects in cache (field values and metadata).
+        Ignite ignite0 = startGrid(0);
+
+        ignite0.active(true);
+
+        IgniteCache cache0 = ignite0.cache(CACHE_NAME);
+
+        cache0.put(0, new TestValue1(0));
+
+        startGrid(1);
+
+        awaitPartitionMapExchange();
+
+        grid(1).cache(CACHE_NAME).put(1, new TestValue2("value"));
+
+        Ignite ignite2 = startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        BinaryObject bo0 = ignite2.binary().builder(DYNAMIC_TYPE_NAME).setField(DYNAMIC_INT_FIELD_NAME, 10).build();
+
+        ignite2.cache(CACHE_NAME).put(2, bo0);
+
+        Ignite ignite3 = startGrid(3);
+
+        awaitPartitionMapExchange();
+
+        BinaryObject bo1 = ignite3
+            .binary()
+            .builder(DYNAMIC_TYPE_NAME)
+            .setField(DYNAMIC_INT_FIELD_NAME, 20)
+            .setField(DYNAMIC_STR_FIELD_NAME, "str")
+            .build();
+
+        ignite3.cache(CACHE_NAME).put(3, bo1);
+
+        stopAllGrids();
+
+        startGrids(4);
+
+        grid(0).active(true);
+
+        examineStaticMetadata(4);
+
+        examineDynamicMetadata(4, contentExaminer0, contentExaminer1, structureExaminer1);
+
+        //2: starts up client node and performs the same set of checks from all nodes including client
+        clientMode = true;
+
+        startGrid(4);
+
+        examineStaticMetadata(5);
+
+        examineDynamicMetadata(5, contentExaminer0, contentExaminer1, structureExaminer1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanIgniteWorkDir();
+    }
+
+    /**
+     *
+     */
+    private void cleanIgniteWorkDir() throws IgniteCheckedException {
+        String baseDir = U.defaultWorkDirectory();
+
+        File baseDirFile = new File(baseDir);
+
+        for (File f : baseDirFile.listFiles())
+            deleteRecursively(U.resolveWorkDirectory(baseDir, f.getName(), false));
+    }
+
+    /**
+     *
+     */
+    private static class TestValue1 {
+        /** */
+        @AffinityKeyMapped
+        private final int val;
+
+        /**
+         * @param val Value.
+         */
+        TestValue1(int val) {
+            this.val = val;
+        }
+
+        /**
+         *
+         */
+        int getValue() {
+            return val;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestValue2 {
+        /** */
+        private final String val;
+
+        /**
+         * @param val Value.
+         */
+        TestValue2(String val) {
+            this.val = val;
+        }
+
+        /**
+         *
+         */
+        String getValue() {
+            return val;
+        }
+    }
+
+    /** */
+    private enum EnumType {
+        /** */ ENUM_VAL_0,
+        /** */ ENUM_VAL_1
+    }
+
+    /**
+     *
+     */
+    private static interface BinaryObjectExaminer {
+        /**
+         * @param cache Cache.
+         */
+        void examine(IgniteCache cache);
+    }
+
+    /** */
+    private BinaryObjectExaminer contentExaminer0 = new BinaryObjectExaminer() {
+        @Override public void examine(IgniteCache cache) {
+            BinaryObject bo = (BinaryObject) cache.get(2);
+
+            int fieldVal = bo.field(DYNAMIC_INT_FIELD_NAME);
+
+            assertEquals(10, fieldVal);
+        }
+    };
+
+    /** */
+    private BinaryObjectExaminer contentExaminer1 = new BinaryObjectExaminer() {
+        @Override public void examine(IgniteCache cache) {
+            BinaryObject bo = (BinaryObject) cache.get(3);
+
+            int fieldVal = bo.field(DYNAMIC_INT_FIELD_NAME);
+
+            assertEquals(20, fieldVal);
+            assertEquals("str", bo.field(DYNAMIC_STR_FIELD_NAME));
+        }
+    };
+
+    /** */
+    private BinaryObjectExaminer structureExaminer0 = new BinaryObjectExaminer() {
+        @Override public void examine(IgniteCache cache) {
+            BinaryObject bo = (BinaryObject) cache.get(2);
+
+            BinaryType type = bo.type();
+
+            assertFalse(type.isEnum());
+
+            assertEquals(DYNAMIC_TYPE_NAME, type.typeName());
+
+            Collection<String> fieldNames = type.fieldNames();
+
+            assertEquals(1, fieldNames.size());
+
+            assertTrue(fieldNames.contains(DYNAMIC_INT_FIELD_NAME));
+
+            assertEquals(DYNAMIC_INT_FIELD_NAME, type.affinityKeyFieldName());
+        }
+    };
+
+    /** */
+    private BinaryObjectExaminer structureExaminer1 = new BinaryObjectExaminer() {
+        @Override public void examine(IgniteCache cache) {
+            BinaryObject bo = (BinaryObject) cache.get(2);
+
+            BinaryType type = bo.type();
+
+            assertFalse(bo.type().isEnum());
+
+            assertEquals(DYNAMIC_TYPE_NAME, type.typeName());
+
+            Collection<String> fieldNames = type.fieldNames();
+
+            assertEquals(2, fieldNames.size());
+
+            assertTrue(fieldNames.contains(DYNAMIC_INT_FIELD_NAME));
+            assertTrue(fieldNames.contains(DYNAMIC_STR_FIELD_NAME));
+        }
+    };
+
+    /** */
+    private BinaryObjectExaminer enumExaminer0 = new BinaryObjectExaminer() {
+        @Override public void examine(IgniteCache cache) {
+            BinaryObject enumBo = (BinaryObject) cache.get(4);
+
+            assertEquals(EnumType.ENUM_VAL_0.ordinal(), enumBo.enumOrdinal());
+
+            BinaryType type = enumBo.type();
+
+            assertTrue(type.isEnum());
+
+            assertEquals(EnumType.class.getName(), type.typeName());
+
+            Collection<BinaryObject> enumVals = type.enumValues();
+
+            assertEquals(2, enumVals.size());
+
+            int i = 0;
+
+            for (BinaryObject bo : enumVals) {
+                assertEquals(i, bo.enumOrdinal());
+
+                assertEquals("ENUM_VAL_" + (i++), bo.enumName());
+            }
+        }
+    };
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/92f66b23/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index fc828a8..01b27ad 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -537,6 +537,7 @@ public abstract class GridAbstractTest extends TestCase {
     protected void beforeTestsStarted() throws Exception {
         // Will clean and re-create marshaller directory from scratch.
         U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", true);
+        U.resolveWorkDirectory(U.defaultWorkDirectory(), "binary_meta", true);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/92f66b23/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index eda99ce..bb9c9d1 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -18,6 +18,7 @@ package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsAtomicCacheRebalancingTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsBinaryMetadataOnClusterRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
@@ -49,6 +50,8 @@ public class IgnitePdsWithIndexingCoreTestSuite extends TestSuite {
         suite.addTestSuite(IgnitePdsAtomicCacheRebalancingTest.class);
         suite.addTestSuite(IgnitePdsTxCacheRebalancingTest.class);
 
+        suite.addTestSuite(IgnitePdsBinaryMetadataOnClusterRestartTest.class);
+
         return suite;
     }
 }