You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/08/21 08:48:26 UTC
[08/45] incubator-ignite git commit: ignite-1258: open sourced
portables implementation
ignite-1258: open sourced portables implementation
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/1f2be19d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/1f2be19d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/1f2be19d
Branch: refs/heads/ignite-1258
Commit: 1f2be19d4cafd94687500618f16c85c873210238
Parents: fcd9d61
Author: Denis Magda <dm...@gridgain.com>
Authored: Tue Aug 18 12:31:26 2015 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Tue Aug 18 12:31:26 2015 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/IgnitePortables.java | 364 +++
.../ignite/internal/GridKernalContextImpl.java | 3 +-
.../apache/ignite/internal/IgniteKernal.java | 3 +-
.../portable/GridGridPortableReaderImpl.java | 2972 ++++++++++++++++++
.../portable/GridPortableAbstractLazyValue.java | 57 +
.../portable/GridPortableBuilderEnum.java | 114 +
.../portable/GridPortableBuilderImpl.java | 519 +++
.../portable/GridPortableBuilderReader.java | 775 +++++
.../GridPortableBuilderSerializationAware.java | 29 +
.../portable/GridPortableBuilderSerializer.java | 210 ++
.../portable/GridPortableClassDescriptor.java | 1344 ++++++++
.../internal/portable/GridPortableContext.java | 1089 +++++++
.../GridPortableEnumArrayLazyValue.java | 111 +
.../portable/GridPortableLazyArrayList.java | 156 +
.../portable/GridPortableLazyLinkedList.java | 210 ++
.../internal/portable/GridPortableLazyMap.java | 214 ++
.../portable/GridPortableLazyMapEntry.java | 66 +
.../internal/portable/GridPortableLazySet.java | 89 +
.../portable/GridPortableLazyValue.java | 28 +
.../portable/GridPortableMarshaller.java | 304 ++
.../portable/GridPortableMetaDataCollector.java | 253 ++
.../portable/GridPortableMetaDataHandler.java | 43 +
.../portable/GridPortableMetaDataImpl.java | 140 +
.../GridPortableObjectArrayLazyValue.java | 89 +
.../internal/portable/GridPortableObjectEx.java | 213 ++
.../portable/GridPortableObjectImpl.java | 383 +++
.../portable/GridPortableObjectOffheapImpl.java | 238 ++
.../portable/GridPortablePlainLazyValue.java | 47 +
.../GridPortablePlainPortableObject.java | 50 +
.../portable/GridPortablePrimitives.java | 773 +++++
.../portable/GridPortableRawReader.java | 33 +
.../portable/GridPortableRawWriter.java | 44 +
.../portable/GridPortableReaderContext.java | 83 +
.../GridPortableThreadLocalMemoryAllocator.java | 163 +
.../internal/portable/GridPortableUtils.java | 380 +++
.../portable/GridPortableValueWithType.java | 74 +
.../portable/GridPortableWriterImpl.java | 1770 +++++++++++
.../ignite/internal/portable/package-info.java | 22 +
.../GridPortableAbstractInputStream.java | 344 ++
.../GridPortableAbstractOutputStream.java | 325 ++
.../streams/GridPortableAbstractStream.java | 83 +
.../streams/GridPortableHeapInputStream.java | 134 +
.../streams/GridPortableHeapOutputStream.java | 155 +
.../streams/GridPortableMemoryAllocator.java | 76 +
.../streams/GridPortableOffheapInputStream.java | 129 +
.../GridPortableOffheapOutputStream.java | 169 +
.../GridPortableSimpleMemoryAllocator.java | 67 +
.../internal/portable/streams/package-info.java | 22 +
.../CacheDefaultPortableAffinityKeyMapper.java | 51 +
.../portable/CacheObjectPortableContext.java | 164 +
.../portable/CacheObjectPortableProcessor.java | 101 +
.../CacheObjectPortableProcessorImpl.java | 949 ++++++
.../cache/portable/IgnitePortablesImpl.java | 176 ++
.../cache/portable/PortableMetaDataKey.java | 80 +
.../processors/cache/portable/package-info.java | 22 +
.../marshaller/portable/PortableMarshaller.java | 348 ++
.../marshaller/portable/package-info.java | 22 +
.../apache/ignite/portable/PortableBuilder.java | 137 +
.../ignite/portable/PortableException.java | 58 +
.../ignite/portable/PortableIdMapper.java | 47 +
.../portable/PortableInvalidClassException.java | 50 +
.../ignite/portable/PortableMarshalAware.java | 40 +
.../ignite/portable/PortableMetadata.java | 53 +
.../apache/ignite/portable/PortableObject.java | 160 +
.../portable/PortableProtocolVersion.java | 32 +
.../ignite/portable/PortableRawReader.java | 241 ++
.../ignite/portable/PortableRawWriter.java | 210 ++
.../apache/ignite/portable/PortableReader.java | 294 ++
.../ignite/portable/PortableSerializer.java | 39 +
.../portable/PortableTypeConfiguration.java | 189 ++
.../apache/ignite/portable/PortableWriter.java | 257 ++
.../apache/ignite/portable/package-info.java | 22 +
72 files changed, 18699 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1f2be19d/modules/core/src/main/java/org/apache/ignite/IgnitePortables.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgnitePortables.java b/modules/core/src/main/java/org/apache/ignite/IgnitePortables.java
new file mode 100644
index 0000000..efb6e6a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/IgnitePortables.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite;
+
+import org.apache.ignite.internal.processors.cache.*;
+import org.apache.ignite.portable.*;
+import org.apache.ignite.marshaller.portable.*;
+
+import org.jetbrains.annotations.*;
+
+import java.sql.*;
+import java.util.*;
+import java.util.Date;
+
+/**
+ * Defines portable objects functionality. With portable objects you are able to:
+ * <ul>
+ * <li>Seamlessly interoperate between Java, .NET, and C++.</li>
+ * <li>Make any object portable with zero code change to your existing code.</li>
+ * <li>Nest portable objects within each other.</li>
+ * <li>Automatically handle {@code circular} or {@code null} references.</li>
+ * <li>Automatically convert collections and maps between Java, .NET, and C++.</li>
+ * <li>
+ * Optionally avoid deserialization of objects on the server side
+ * (objects are stored in {@link PortableObject} format).
+ * </li>
+ * <li>Avoid need to have concrete class definitions on the server side.</li>
+ * <li>Dynamically change structure of the classes without having to restart the cluster.</li>
+ * <li>Index into portable objects for querying purposes.</li>
+ * </ul>
+ * <h1 class="header">Working With Portables Directly</h1>
+ * Once an object is defined as portable,
+ * Ignite will always store it in memory in the portable (i.e. binary) format.
+ * User can choose to work either with the portable format or with the deserialized form
+ * (assuming that class definitions are present in the classpath).
+ * <p>
+ * To work with the portable format directly, user should create a special cache projection
+ * using {@link IgniteInternalCache#keepPortable()} method and then retrieve individual fields as needed:
+ * <pre name=code class=java>
+ * CacheProjection<GridPortableObject.class, GridPortableObject.class> prj = cache.keepPortable();
+ *
+ * // Convert instance of MyKey to portable format.
+ * // We could also use GridPortableBuilder to create
+ * // the key in portable format directly.
+ * GridPortableObject key = grid.portables().toPortable(new MyKey());
+ *
+ * GridPortableObject val = prj.get(key);
+ *
+ * String field = val.field("myFieldName");
+ * </pre>
+ * Alternatively, if we have class definitions in the classpath, we may choose to work with deserialized
+ * typed objects at all times. In this case we do incur the deserialization cost, however,
+ * Ignite will only deserialize on the first access and will cache the deserialized object,
+ * so it does not have to be deserialized again:
+ * <pre name=code class=java>
+ * CacheProjection<MyKey.class, MyValue.class> prj =
+ * cache.projection(MyKey.class, MyValue.class);
+ *
+ * MyValue val = prj.get(new MyKey());
+ *
+ * // Normal java getter.
+ * String fieldVal = val.getMyFieldName();
+ * </pre>
+ * If we used, for example, one of the automatically handled portable types for a key, like integer,
+ * and still wanted to work with binary portable format for values, then we would declare cache projection
+ * as follows:
+ * <pre name=code class=java>
+ * CacheProjection<Integer.class, GridPortableObject.class> prj = cache.keepPortable();
+ * </pre>
+ * <h1 class="header">Automatic Portable Types</h1>
+ * Note that only portable classes are converted to {@link PortableObject} format. Following
+ * classes are never converted (e.g., {@link #toPortable(Object)} method will return original
+ * object, and instances of these classes will be stored in cache without changes):
+ * <ul>
+ * <li>All primitives (byte, int, ...) and there boxed versions (Byte, Integer, ...)</li>
+ * <li>Arrays of primitives (byte[], int[], ...)</li>
+ * <li>{@link String} and array of {@link String}s</li>
+ * <li>{@link UUID} and array of {@link UUID}s</li>
+ * <li>{@link Date} and array of {@link Date}s</li>
+ * <li>{@link Timestamp} and array of {@link Timestamp}s</li>
+ * <li>Enums and array of enums</li>
+ * <li>
+ * Maps, collections and array of objects (but objects inside
+ * them will still be converted if they are portable)
+ * </li>
+ * </ul>
+ * <h1 class="header">Working With Maps and Collections</h1>
+ * All maps and collections in the portable objects are serialized automatically. When working
+ * with different platforms, e.g. C++ or .NET, Ignite will automatically pick the most
+ * adequate collection or map in either language. For example, {@link ArrayList} in Java will become
+ * {@code List} in C#, {@link LinkedList} in Java is {@link LinkedList} in C#, {@link HashMap}
+ * in Java is {@code Dictionary} in C#, and {@link TreeMap} in Java becomes {@code SortedDictionary}
+ * in C#, etc.
+ * <h1 class="header">Building Portable Objects</h1>
+ * Ignite comes with {@link PortableBuilder} which allows to build portable objects dynamically:
+ * <pre name=code class=java>
+ * GridPortableBuilder builder = Ignition.ignite().portables().builder();
+ *
+ * builder.typeId("MyObject");
+ *
+ * builder.stringField("fieldA", "A");
+ * build.intField("fieldB", "B");
+ *
+ * GridPortableObject portableObj = builder.build();
+ * </pre>
+ * For the cases when class definition is present
+ * in the class path, it is also possible to populate a standard POJO and then
+ * convert it to portable format, like so:
+ * <pre name=code class=java>
+ * MyObject obj = new MyObject();
+ *
+ * obj.setFieldA("A");
+ * obj.setFieldB(123);
+ *
+ * GridPortableObject portableObj = Ignition.ignite().portables().toPortable(obj);
+ * </pre>
+ * NOTE: you don't need to convert typed objects to portable format before storing
+ * them in cache, Ignite will do that automatically.
+ * <h1 class="header">Portable Metadata</h1>
+ * Even though Ignite portable protocol only works with hash codes for type and field names
+ * to achieve better performance, Ignite provides metadata for all portable types which
+ * can be queried ar runtime via any of the {@link IgnitePortables#metadata(Class) GridPortables.metadata(...)}
+ * methods. Having metadata also allows for proper formatting of {@code GridPortableObject.toString()} method,
+ * even when portable objects are kept in binary format only, which may be necessary for audit reasons.
+ * <h1 class="header">Dynamic Structure Changes</h1>
+ * Since objects are always cached in the portable binary format, server does not need to
+ * be aware of the class definitions. Moreover, if class definitions are not present or not
+ * used on the server, then clients can continuously change the structure of the portable
+ * objects without having to restart the cluster. For example, if one client stores a
+ * certain class with fields A and B, and another client stores the same class with
+ * fields B and C, then the server-side portable object will have the fields A, B, and C.
+ * As the structure of a portable object changes, the new fields become available for SQL queries
+ * automatically.
+ * <h1 class="header">Configuration</h1>
+ * By default all your objects are considered as portables and no specific configuration is needed.
+ * However, in some cases, like when an object is used by both Java and .Net, you may need to specify portable objects
+ * explicitly by calling {@link PortableMarshaller#setClassNames(Collection)}.
+ * The only requirement Ignite imposes is that your object has an empty
+ * constructor. Note, that since server side does not have to know the class definition,
+ * you only need to list portable objects in configuration on the client side. However, if you
+ * list them on the server side as well, then you get the ability to deserialize portable objects
+ * into concrete types on the server as well as on the client.
+ * <p>
+ * Here is an example of portable configuration (note that star (*) notation is supported):
+ * <pre name=code class=xml>
+ * ...
+ * <!-- Explicit portable objects configuration. -->
+ * <property name="marshaller">
+ * <bean class="org.gridgain.grid.marshaller.portable.PortableMarshaller">
+ * <property name="classNames">
+ * <list>
+ * <value>my.package.for.portable.objects.*</value>
+ * <value>org.apache.ignite.examples.client.portable.Employee</value>
+ * </list>
+ * </property>
+ * </bean>
+ * </property>
+ * ...
+ * </pre>
+ * or from code:
+ * <pre name=code class=java>
+ * IgniteConfiguration cfg = new IgniteConfiguration();
+ *
+ * PortableMarshaller marsh = new PortableMarshaller();
+ *
+ * marsh.setClassNames(Arrays.asList(
+ * Employee.class.getName(),
+ * Address.class.getName())
+ * );
+ *
+ * cfg.setMarshaller(marsh);
+ * </pre>
+ * You can also specify class name for a portable object via {@link PortableTypeConfiguration}.
+ * Do it in case if you need to override other configuration properties on per-type level, like
+ * ID-mapper, or serializer.
+ * <h1 class="header">Custom Affinity Keys</h1>
+ * Often you need to specify an alternate key (not the cache key) for affinity routing whenever
+ * storing objects in cache. For example, if you are caching {@code Employee} object with
+ * {@code Organization}, and want to colocate employees with organization they work for,
+ * so you can process them together, you need to specify an alternate affinity key.
+ * With portable objects you would have to do it as following:
+ * <pre name=code class=xml>
+ * <property name="marshaller">
+ * <bean class="org.gridgain.grid.marshaller.portable.PortableMarshaller">
+ * ...
+ * <property name="typeConfigurations">
+ * <list>
+ * <bean class="org.apache.ignite.portables.PortableTypeConfiguration">
+ * <property name="className" value="org.apache.ignite.examples.client.portable.EmployeeKey"/>
+ * <property name="affinityKeyFieldName" value="organizationId"/>
+ * </bean>
+ * </list>
+ * </property>
+ * ...
+ * </bean>
+ * </property>
+ * </pre>
+ * <h1 class="header">Serialization</h1>
+ * Serialization and deserialization works out-of-the-box in Ignite. However, you can provide your own custom
+ * serialization logic by optionally implementing {@link PortableMarshalAware} interface, like so:
+ * <pre name=code class=java>
+ * public class Address implements GridPortableMarshalAware {
+ * private String street;
+ * private int zip;
+ *
+ * // Empty constructor required for portable deserialization.
+ * public Address() {}
+ *
+ * @Override public void writePortable(GridPortableWriter writer) throws GridPortableException {
+ * writer.writeString("street", street);
+ * writer.writeInt("zip", zip);
+ * }
+ *
+ * @Override public void readPortable(GridPortableReader reader) throws GridPortableException {
+ * street = reader.readString("street");
+ * zip = reader.readInt("zip");
+ * }
+ * }
+ * </pre>
+ * Alternatively, if you cannot change class definitions, you can provide custom serialization
+ * logic in {@link PortableSerializer} either globally in {@link PortableMarshaller} or
+ * for a specific type via {@link PortableTypeConfiguration} instance.
+ * <p>
+ * Similar to java serialization you can use {@code writeReplace()} and {@code readResolve()} methods.
+ * <ul>
+ * <li>
+ * {@code readResolve} is defined as follows: {@code ANY-ACCESS-MODIFIER Object readResolve()}.
+ * It may be used to replace the de-serialized object by another one of your choice.
+ * </li>
+ * <li>
+ * {@code writeReplace} is defined as follows: {@code ANY-ACCESS-MODIFIER Object writeReplace()}. This method
+ * allows the developer to provide a replacement object that will be serialized instead of the original one.
+ * </li>
+ * </ul>
+ *
+ * <h1 class="header">Custom ID Mappers</h1>
+ * Ignite implementation uses name hash codes to generate IDs for class names or field names
+ * internally. However, in cases when you want to provide your own ID mapping schema,
+ * you can provide your own {@link PortableIdMapper} implementation.
+ * <p>
+ * ID-mapper may be provided either globally in {@link PortableMarshaller},
+ * or for a specific type via {@link PortableTypeConfiguration} instance.
+ * <h1 class="header">Query Indexing</h1>
+ * Portable objects can be indexed for querying by specifying index fields in
+ * {@link org.apache.ignite.cache.CacheTypeMetadata} inside of specific
+ * {@link org.apache.ignite.configuration.CacheConfiguration} instance,
+ * like so:
+ * <pre name=code class=xml>
+ * ...
+ * <bean class="org.apache.ignite.cache.CacheConfiguration">
+ * ...
+ * <property name="typeMetadata">
+ * <list>
+ * <bean class="CacheTypeMetadata">
+ * <property name="type" value="Employee"/>
+ *
+ * <!-- Fields to index in ascending order. -->
+ * <property name="ascendingFields">
+ * <map>
+ * <entry key="name" value="java.lang.String"/>
+ *
+ * <!-- Nested portable objects can also be indexed. -->
+ * <entry key="address.zip" value="java.lang.Integer"/>
+ * </map>
+ * </property>
+ * </bean>
+ * </list>
+ * </property>
+ * </bean>
+ * </pre>
+ */
+public interface IgnitePortables {
+ /**
+ * Gets type ID for given type name.
+ *
+ * @param typeName Type name.
+ * @return Type ID.
+ */
+ public int typeId(String typeName);
+
+ /**
+ * Converts provided object to instance of {@link PortableObject}.
+ *
+ * @param obj Object to convert.
+ * @return Converted object.
+ * @throws PortableException In case of error.
+ */
+ public <T> T toPortable(@Nullable Object obj) throws PortableException;
+
+ /**
+ * Creates new portable builder.
+ *
+ * @param typeId ID of the type.
+ * @return Newly portable builder.
+ */
+ public PortableBuilder builder(int typeId);
+
+ /**
+ * Creates new portable builder.
+ *
+ * @param typeName Type name.
+ * @return Newly portable builder.
+ */
+ public PortableBuilder builder(String typeName);
+
+ /**
+ * Creates portable builder initialized by existing portable object.
+ *
+ * @param portableObj Portable object to initialize builder.
+ * @return Portable builder.
+ */
+ public PortableBuilder builder(PortableObject portableObj);
+
+ /**
+ * Gets metadata for provided class.
+ *
+ * @param cls Class.
+ * @return Metadata.
+ * @throws PortableException In case of error.
+ */
+ @Nullable public PortableMetadata metadata(Class<?> cls) throws PortableException;
+
+ /**
+ * Gets metadata for provided class name.
+ *
+ * @param typeName Type name.
+ * @return Metadata.
+ * @throws PortableException In case of error.
+ */
+ @Nullable public PortableMetadata metadata(String typeName) throws PortableException;
+
+ /**
+ * Gets metadata for provided type ID.
+ *
+ * @param typeId Type ID.
+ * @return Metadata.
+ * @throws PortableException In case of error.
+ */
+ @Nullable public PortableMetadata metadata(int typeId) throws PortableException;
+
+ /**
+ * Gets metadata for all known types.
+ *
+ * @return Metadata.
+ * @throws PortableException In case of error.
+ */
+ public Collection<PortableMetadata> metadata() throws PortableException;
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1f2be19d/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index fd8b50c..01dadfd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -32,6 +32,7 @@ import org.apache.ignite.internal.managers.loadbalancer.*;
import org.apache.ignite.internal.managers.swapspace.*;
import org.apache.ignite.internal.processors.affinity.*;
import org.apache.ignite.internal.processors.cache.*;
+import org.apache.ignite.internal.processors.cache.portable.*;
import org.apache.ignite.internal.processors.cacheobject.*;
import org.apache.ignite.internal.processors.clock.*;
import org.apache.ignite.internal.processors.closure.*;
@@ -803,7 +804,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
return res;
if (cls.equals(IgniteCacheObjectProcessor.class))
- return (T)new IgniteCacheObjectProcessorImpl(this);
+ return (T)new CacheObjectPortableProcessorImpl(this);
if (cls.equals(CacheConflictResolutionManager.class))
return null;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1f2be19d/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index bf47f63..e32d5e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.managers.swapspace.*;
import org.apache.ignite.internal.processors.*;
import org.apache.ignite.internal.processors.affinity.*;
import org.apache.ignite.internal.processors.cache.*;
+import org.apache.ignite.internal.processors.cache.portable.*;
import org.apache.ignite.internal.processors.cacheobject.*;
import org.apache.ignite.internal.processors.clock.*;
import org.apache.ignite.internal.processors.closure.*;
@@ -2960,7 +2961,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
return comp;
if (cls.equals(IgniteCacheObjectProcessor.class))
- return (T)new IgniteCacheObjectProcessorImpl(ctx);
+ return (T)new CacheObjectPortableProcessorImpl(ctx);
if (cls.equals(DiscoveryNodeValidationProcessor.class))
return (T)new OsDiscoveryNodeValidationProcessor(ctx);