You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/07/01 10:40:51 UTC
[01/20] ignite git commit: IGNITE-3184: Hadoop: HADOOP_HOME is no
longer required is all HADOOP_*_HOME environment variables are set.
Repository: ignite
Updated Branches:
refs/heads/ignite-3410 [created] 172764757
IGNITE-3184: Hadoop: HADOOP_HOME is no longer required is all HADOOP_*_HOME environment variables are set.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9007fb6f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9007fb6f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9007fb6f
Branch: refs/heads/ignite-3410
Commit: 9007fb6fb54175722f38a36472d59cfa0b65ef33
Parents: 46ba701
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Mon Jun 27 13:44:38 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Mon Jun 27 13:44:38 2016 +0300
----------------------------------------------------------------------
bin/include/hadoop-classpath.bat | 73 ------
bin/include/hadoop-classpath.sh | 65 ------
bin/include/setenv.bat | 6 +-
bin/include/setenv.sh | 25 +-
.../processors/hadoop/HadoopClassLoader.java | 73 +-----
.../processors/hadoop/HadoopClasspathMain.java | 44 ++++
.../processors/hadoop/HadoopClasspathUtils.java | 233 +++++++++++++++++++
.../processors/hadoop/HadoopLocations.java | 78 +++++++
.../processors/hadoop/HadoopProcessor.java | 47 ++--
9 files changed, 404 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9007fb6f/bin/include/hadoop-classpath.bat
----------------------------------------------------------------------
diff --git a/bin/include/hadoop-classpath.bat b/bin/include/hadoop-classpath.bat
deleted file mode 100644
index dd293a7..0000000
--- a/bin/include/hadoop-classpath.bat
+++ /dev/null
@@ -1,73 +0,0 @@
-::
-:: Licensed to the Apache Software Foundation (ASF) under one or more
-:: contributor license agreements. See the NOTICE file distributed with
-:: this work for additional information regarding copyright ownership.
-:: The ASF licenses this file to You under the Apache License, Version 2.0
-:: (the "License"); you may not use this file except in compliance with
-:: the License. You may obtain a copy of the License at
-::
-:: http://www.apache.org/licenses/LICENSE-2.0
-::
-:: Unless required by applicable law or agreed to in writing, software
-:: distributed under the License is distributed on an "AS IS" BASIS,
-:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-:: See the License for the specific language governing permissions and
-:: limitations under the License.
-::
-
-:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-:: Hadoop class path resolver.
-:: Requires environment variables 'HADOOP_PREFIX' or 'HADOOP_HOME'
-:: to be set.
-:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-
-:: Turn off script echoing.
-@echo off
-
-:: Check if environment passes deprecated HADOOP_HOME.
-if not defined HADOOP_PREFIX set HADOOP_PREFIX=%HADOOP_HOME%
-
-:: Exit if we cannot find Hadoop installation directory.
-if not defined HADOOP_PREFIX goto :eof
-
-:: Trim quotes.
-set HADOOP_PREFIX=%HADOOP_PREFIX:"=%
-
-:: Trim slashes.
-if %HADOOP_PREFIX:~-1,1% == \ (
- set HADOOP_PREFIX=%HADOOP_PREFIX:~0,-1%
-)
-
-::
-:: Setting the rest of Hadoop environment variables.
-::
-
-if not defined HADOOP_COMMON_HOME set HADOOP_COMMON_HOME=%HADOOP_PREFIX%\share\hadoop\common
-if not defined HADOOP_HDFS_HOME set HADOOP_HDFS_HOME=%HADOOP_PREFIX%\share\hadoop\hdfs
-if not defined HADOOP_MAPRED_HOME set HADOOP_MAPRED_HOME=%HADOOP_PREFIX%\share\hadoop\mapreduce
-
-::
-:: Libraries included in classpath.
-::
-
-set CP=%HADOOP_COMMON_HOME%\lib\*;%HADOOP_MAPRED_HOME%\lib\*;%HADOOP_MAPRED_HOME%\lib\*
-
-:: hadoop-auth-* jar can be located either in home or in home/lib directory, depending on the hadoop version.
-for /f %%f in ('dir /B %HADOOP_COMMON_HOME%\hadoop-auth-* ^>nul 2^>^&1') do call :concat %HADOOP_COMMON_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_COMMON_HOME%\lib\hadoop-auth-* ^>nul 2^>^&1') do call :concat %HADOOP_COMMON_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_COMMON_HOME%\hadoop-common-*') do call :concat %HADOOP_COMMON_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_HDFS_HOME%\hadoop-hdfs-*') do call :concat %HADOOP_HDFS_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_MAPRED_HOME%\hadoop-mapreduce-client-common-*') do call :concat %HADOOP_MAPRED_HOME%\%%f
-for /f %%f in ('dir /B %HADOOP_MAPRED_HOME%\hadoop-mapreduce-client-core-*') do call :concat %HADOOP_MAPRED_HOME%\%%f
-
-:: Export result.
-set IGNITE_HADOOP_CLASSPATH=%CP%
-
-:: Exit.
-goto :eof
-
-:: Function that adds jar dependency into classpath.
-:concat
- set file=%1
- if %file:~-9,9% neq tests.jar set CP=%CP%;%1
-goto :eof
http://git-wip-us.apache.org/repos/asf/ignite/blob/9007fb6f/bin/include/hadoop-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/include/hadoop-classpath.sh b/bin/include/hadoop-classpath.sh
deleted file mode 100755
index 3a0aeb1..0000000
--- a/bin/include/hadoop-classpath.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-####################################################################
-# Hadoop class path resolver.
-# Requires environment variables 'HADOOP_PREFIX' or 'HADOOP_HOME'
-# to be set. If they are both undefined , tries to read them from
-# from '/etc/default/hadoop' file. The final results are printed
-# into standard output.
-####################################################################
-
-# Resolve constants.
-HADOOP_DEFAULTS="/etc/default/hadoop"
-HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_HOME}
-
-# Try get all variables from Hadoop default environment config
-# if they have not been passed into the script.
-if [[ -z "$HADOOP_PREFIX" && -f "$HADOOP_DEFAULTS" ]]; then
- source "$HADOOP_DEFAULTS"
-fi
-
-# Return if Hadoop couldn't be found.
-[ -z "$HADOOP_PREFIX" ] && return
-
-#
-# Resolve the rest of Hadoop environment variables.
-#
-
-HADOOP_COMMON_HOME=${HADOOP_COMMON_HOME-"${HADOOP_PREFIX}/share/hadoop/common"}
-HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME-"${HADOOP_PREFIX}/share/hadoop/hdfs"}
-HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME-"${HADOOP_PREFIX}/share/hadoop/mapreduce"}
-
-#
-# Calculate classpath string with required Hadoop libraries.
-#
-
-# Add all Hadoop libs.
-IGNITE_HADOOP_CLASSPATH="${HADOOP_COMMON_HOME}/lib/*${SEP}${HADOOP_MAPRED_HOME}/lib/*${SEP}${HADOOP_MAPRED_HOME}/lib/*"
-
-# Skip globbing pattern if it cannot be resolved.
-shopt -s nullglob
-
-# Add jars to classpath excluding tests.
-# hadoop-auth-* jar can be located either in home or in home/lib directory, depending on the hadoop version.
-for file in ${HADOOP_HDFS_HOME}/hadoop-hdfs-* \
- ${HADOOP_COMMON_HOME}/hadoop-{common,auth}-* \
- ${HADOOP_COMMON_HOME}/lib/hadoop-auth-* \
- ${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-{common,core}-*; do
- [[ "$file" != *-tests.jar ]] && IGNITE_HADOOP_CLASSPATH=${IGNITE_HADOOP_CLASSPATH}${SEP}${file}
-done
http://git-wip-us.apache.org/repos/asf/ignite/blob/9007fb6f/bin/include/setenv.bat
----------------------------------------------------------------------
diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat
index 811bc95..dec1ee5 100644
--- a/bin/include/setenv.bat
+++ b/bin/include/setenv.bat
@@ -47,11 +47,9 @@ if exist %IGNITE_HOME%\libs\ignite-hadoop set HADOOP_EDITION=1
if defined USER_LIBS set IGNITE_LIBS=%USER_LIBS%;%IGNITE_LIBS%
-if "%HADOOP_EDITION%" == "1" call "%SCRIPTS_HOME%\include\hadoop-classpath.bat"
+FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\ignite-hadoop\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i
-set COMMON_HOME_LIB=%HADOOP_COMMON_HOME%\lib
-
-if "%IGNITE_HADOOP_CLASSPATH%" == "" goto :eof
+if "[%IGNITE_HADOOP_CLASSPATH%]" == "[]" exit 1
set IGNITE_LIBS=%IGNITE_LIBS%;%IGNITE_HADOOP_CLASSPATH%
http://git-wip-us.apache.org/repos/asf/ignite/blob/9007fb6f/bin/include/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh
index 74b6608..a85cba3 100755
--- a/bin/include/setenv.sh
+++ b/bin/include/setenv.sh
@@ -72,9 +72,28 @@ if [ "${USER_LIBS}" != "" ]; then
fi
if [ "${HADOOP_EDITION}" == "1" ]; then
- . "${SCRIPTS_HOME}"/include/hadoop-classpath.sh
+ # Resolve constants.
+ HADOOP_DEFAULTS="/etc/default/hadoop"
- if [ "${IGNITE_HADOOP_CLASSPATH}" != "" ]; then
- IGNITE_LIBS=${IGNITE_LIBS}${SEP}$IGNITE_HADOOP_CLASSPATH
+ #
+ # Resolve the rest of Hadoop environment variables.
+ #
+ if [[ -z "${HADOOP_COMMON_HOME}" || -z "${HADOOP_HDFS_HOME}" || -z "${HADOOP_MAPRED_HOME}" ]]; then
+ if [ -f "$HADOOP_DEFAULTS" ]; then
+ source "$HADOOP_DEFAULTS"
+ fi
fi
+
+ IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/ignite-hadoop/'*' \
+ org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ":" )
+
+ statusCode=${?}
+
+ if [ "${statusCode}" -ne 0 ]; then
+ exit ${statusCode}
+ fi
+
+ unset statusCode
+
+ IGNITE_LIBS=${IGNITE_LIBS}${SEP}${IGNITE_HADOOP_CLASSPATH}
fi
http://git-wip-us.apache.org/repos/asf/ignite/blob/9007fb6f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 340b35b..1c844c4 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -17,11 +17,8 @@
package org.apache.ignite.internal.processors.hadoop;
-import java.io.File;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
-import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
@@ -460,40 +457,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
}
/**
- * @param name Variable name.
- * @param dflt Default.
- * @return Value.
- */
- private static String getEnv(String name, String dflt) {
- String res = System.getProperty(name);
-
- if (F.isEmpty(res))
- res = System.getenv(name);
-
- return F.isEmpty(res) ? dflt : res;
- }
-
- /**
- * @param res Result.
- * @param dir Directory.
- * @param startsWith Starts with prefix.
- * @throws MalformedURLException If failed.
- */
- private static void addUrls(Collection<URL> res, File dir, final String startsWith) throws Exception {
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return startsWith == null || name.startsWith(startsWith);
- }
- });
-
- if (files == null)
- throw new IOException("Path is not a directory: " + dir);
-
- for (File file : files)
- res.add(file.toURI().toURL());
- }
-
- /**
* @param urls URLs.
* @return URLs.
*/
@@ -519,13 +482,6 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
}
/**
- * @return HADOOP_HOME Variable.
- */
- @Nullable public static String hadoopHome() {
- return getEnv("HADOOP_PREFIX", getEnv("HADOOP_HOME", null));
- }
-
- /**
* @return Collection of jar URLs.
* @throws IgniteCheckedException If failed.
*/
@@ -541,34 +497,11 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
if (hadoopUrls != null)
return hadoopUrls;
- hadoopUrls = new ArrayList<>();
-
- String hadoopPrefix = hadoopHome();
-
- if (F.isEmpty(hadoopPrefix))
- throw new IgniteCheckedException("Failed resolve Hadoop installation location. Either HADOOP_PREFIX or " +
- "HADOOP_HOME environment variables must be set.");
-
- String commonHome = getEnv("HADOOP_COMMON_HOME", hadoopPrefix + "/share/hadoop/common");
- String hdfsHome = getEnv("HADOOP_HDFS_HOME", hadoopPrefix + "/share/hadoop/hdfs");
- String mapredHome = getEnv("HADOOP_MAPRED_HOME", hadoopPrefix + "/share/hadoop/mapreduce");
-
try {
- addUrls(hadoopUrls, new File(commonHome + "/lib"), null);
- addUrls(hadoopUrls, new File(hdfsHome + "/lib"), null);
- addUrls(hadoopUrls, new File(mapredHome + "/lib"), null);
-
- addUrls(hadoopUrls, new File(hdfsHome), "hadoop-hdfs-");
-
- addUrls(hadoopUrls, new File(commonHome), "hadoop-common-");
- addUrls(hadoopUrls, new File(commonHome), "hadoop-auth-");
- addUrls(hadoopUrls, new File(commonHome + "/lib"), "hadoop-auth-");
-
- addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-common");
- addUrls(hadoopUrls, new File(mapredHome), "hadoop-mapreduce-client-core");
+ hadoopUrls = HadoopClasspathUtils.classpathUrls();
}
- catch (Exception e) {
- throw new IgniteCheckedException(e);
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations.", e);
}
hadoopJars = hadoopUrls;
http://git-wip-us.apache.org/repos/asf/ignite/blob/9007fb6f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
new file mode 100644
index 0000000..5279b7d
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathMain.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Main class to compose Hadoop classpath depending on the environment.
+ * This class is designed to be independent on any Ignite classes as possible.
+ * Please make sure to pass the path separator character as the 1st parameter to the main method.
+ */
+public class HadoopClasspathMain {
+ /**
+ * Main method to be executed from scripts. It prints the classpath to the standard output.
+ *
+ * @param args The 1st argument should be the path separator character (":" on Linux, ";" on Windows).
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length < 1)
+ throw new IllegalArgumentException("Path separator must be passed as the first argument.");
+
+ String separator = args[0];
+
+ StringBuilder sb = new StringBuilder();
+
+ for (String path : HadoopClasspathUtils.classpathForJavaProcess())
+ sb.append(path).append(separator);
+
+ System.out.println(sb);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9007fb6f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
new file mode 100644
index 0000000..986b23e
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Hadoop classpath utilities.
+ */
+public class HadoopClasspathUtils {
+ /**
+ * Gets Hadoop class path as list of classpath elements for process.
+ *
+ * @return List of the class path elements.
+ * @throws IOException If failed.
+ */
+ public static List<String> classpathForJavaProcess() throws IOException {
+ List<String> res = new ArrayList<>();
+
+ for (final SearchDirectory dir : classpathDirectories()) {
+ if (dir.hasFilter()) {
+ for (File file : dir.files())
+ res.add(file.getAbsolutePath());
+ }
+ else
+ res.add(dir.absolutePath() + File.separator + '*');
+ }
+
+ return res;
+ }
+
+ /**
+ * Gets Hadoop class path as a list of URLs (for in-process class loader usage).
+ *
+ * @return List of class path URLs.
+ * @throws IOException If failed.
+ */
+ public static List<URL> classpathUrls() throws IOException {
+ List<URL> res = new ArrayList<>();
+
+ for (SearchDirectory dir : classpathDirectories()) {
+ for (File file : dir.files()) {
+ try {
+ res.add(file.toURI().toURL());
+ }
+ catch (MalformedURLException e) {
+ throw new IOException("Failed to convert file path to URL: " + file.getPath());
+ }
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Gets Hadoop locations.
+ *
+ * @return The Hadoop locations, never null.
+ */
+ public static HadoopLocations hadoopLocations() throws IOException {
+ final String hadoopHome = systemOrEnv("HADOOP_HOME", systemOrEnv("HADOOP_PREFIX", null));
+
+ String commonHome = resolveLocation("HADOOP_COMMON_HOME", hadoopHome, "/share/hadoop/common");
+ String hdfsHome = resolveLocation("HADOOP_HDFS_HOME", hadoopHome, "/share/hadoop/hdfs");
+ String mapredHome = resolveLocation("HADOOP_MAPRED_HOME", hadoopHome, "/share/hadoop/mapreduce");
+
+ return new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
+ }
+
+ /**
+ * Gets base directories to discover classpath elements in.
+ *
+ * @return Collection of directory and mask pairs.
+ * @throws IOException if a mandatory classpath location is not found.
+ */
+ private static Collection<SearchDirectory> classpathDirectories() throws IOException {
+ HadoopLocations loc = hadoopLocations();
+
+ Collection<SearchDirectory> res = new ArrayList<>();
+
+ res.add(new SearchDirectory(new File(loc.commonHome(), "lib"), null));
+ res.add(new SearchDirectory(new File(loc.hdfsHome(), "lib"), null));
+ res.add(new SearchDirectory(new File(loc.mapredHome(), "lib"), null));
+
+ res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-common-"));
+ res.add(new SearchDirectory(new File(loc.commonHome()), "hadoop-auth-"));
+
+ res.add(new SearchDirectory(new File(loc.hdfsHome()), "hadoop-hdfs-"));
+
+ res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-common"));
+ res.add(new SearchDirectory(new File(loc.mapredHome()), "hadoop-mapreduce-client-core"));
+
+ return res;
+ }
+
+ /**
+ * Resolves a Hadoop location directory.
+ *
+ * @param envVarName Environment variable name. The value denotes the location path.
+ * @param hadoopHome Hadoop home location, may be null.
+ * @param expHadoopHomeRelativePath The path relative to Hadoop home, expected to start with path separator.
+ * @throws IOException If the value cannot be resolved to an existing directory.
+ */
+ private static String resolveLocation(String envVarName, String hadoopHome, String expHadoopHomeRelativePath)
+ throws IOException {
+ String val = systemOrEnv(envVarName, null);
+
+ if (val == null) {
+ // The env. variable is not set. Try to resolve the location relative HADOOP_HOME:
+ if (!directoryExists(hadoopHome))
+ throw new IOException("Failed to resolve Hadoop installation location. " +
+ envVarName + " or HADOOP_HOME environment variable should be set.");
+
+ val = hadoopHome + expHadoopHomeRelativePath;
+ }
+
+ if (!directoryExists(val))
+ throw new IOException("Failed to resolve Hadoop location [path=" + val + ']');
+
+ return val;
+ }
+
+ /**
+ * Note that this method does not treat empty value as an absent value.
+ *
+ * @param name Variable name.
+ * @param dflt Default.
+ * @return Value.
+ */
+ private static String systemOrEnv(String name, String dflt) {
+ String res = System.getProperty(name);
+
+ if (res == null)
+ res = System.getenv(name);
+
+ return res == null ? dflt : res;
+ }
+
+ /**
+ * Answers if the given path denotes existing directory.
+ *
+ * @param path The directory path.
+ * @return {@code True} if the given path denotes an existing directory.
+ */
+ private static boolean directoryExists(String path) {
+ if (path == null)
+ return false;
+
+ Path p = Paths.get(path);
+
+ return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p);
+ }
+
+ /**
+ * Simple pair-like structure to hold directory name and a mask assigned to it.
+ */
+ private static class SearchDirectory {
+ /** File. */
+ private final File dir;
+
+ /** The mask. */
+ private final String filter;
+
+ /**
+ * Constructor.
+ *
+ * @param dir Directory.
+ * @param filter Filter.
+ */
+ private SearchDirectory(File dir, String filter) throws IOException {
+ this.dir = dir;
+ this.filter = filter;
+
+ if (!directoryExists(dir.getAbsolutePath()))
+ throw new IOException("Directory cannot be read: " + dir.getAbsolutePath());
+ }
+
+ /**
+ * @return Absolute path.
+ */
+ private String absolutePath() {
+ return dir.getAbsolutePath();
+ }
+
+ /**
+ * @return Child files.
+ */
+ private File[] files() throws IOException {
+ File[] files = dir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return filter == null || name.startsWith(filter);
+ }
+ });
+
+ if (files == null)
+ throw new IOException("Path is not a directory. [dir=" + dir + ']');
+
+ return files;
+ }
+
+ /**
+ * @return {@code True} if filter exists.
+ */
+ private boolean hasFilter() {
+ return filter != null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9007fb6f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
new file mode 100644
index 0000000..b3e2d12
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Simple structure to hold Hadoop directory locations.
+ */
+public class HadoopLocations {
+ /** Hadoop home. */
+ private final String home;
+
+ /** Common home. */
+ private final String commonHome;
+
+ /** HDFS home. */
+ private final String hdfsHome;
+
+ /** Mapred home. */
+ private final String mapredHome;
+
+ /**
+ * Constructor.
+ *
+ * @param home Hadoop home.
+ * @param commonHome Common home.
+ * @param hdfsHome HDFS home.
+ * @param mapredHome Mapred home.
+ */
+ public HadoopLocations(String home, String commonHome, String hdfsHome, String mapredHome) {
+ this.home = home;
+ this.commonHome = commonHome;
+ this.hdfsHome = hdfsHome;
+ this.mapredHome = mapredHome;
+ }
+
+ /**
+ * @return Hadoop home.
+ */
+ public String home() {
+ return home;
+ }
+
+ /**
+ * @return Common home.
+ */
+ public String commonHome() {
+ return commonHome;
+ }
+
+ /**
+ * @return HDFS home.
+ */
+ public String hdfsHome() {
+ return hdfsHome;
+ }
+
+ /**
+ * @return Mapred home.
+ */
+ public String mapredHome() {
+ return mapredHome;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9007fb6f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index 7292c78..c45953e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.hadoop;
+import java.io.IOException;
import java.util.List;
import java.util.ListIterator;
import java.util.concurrent.atomic.AtomicInteger;
@@ -33,9 +34,6 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
-import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.hadoopHome;
-import static org.apache.ignite.internal.processors.hadoop.HadoopClassLoader.hadoopUrls;
-
/**
* Hadoop processor.
*/
@@ -74,36 +72,35 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
validate(cfg);
- if (hadoopHome() != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + hadoopHome());
-
- boolean ok = false;
+ try {
+ HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
- try { // Check for Hadoop installation.
- hadoopUrls();
+ if (loc.home() != null)
+ U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
- ok = true;
+ U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
+ U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
+ U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
}
- catch (IgniteCheckedException e) {
- U.quietAndWarn(log, e.getMessage());
+ catch (IOException ioe) {
+ throw new IgniteCheckedException(ioe);
}
- if (ok) {
- hctx = new HadoopContext(
- ctx,
- cfg,
- new HadoopJobTracker(),
- new HadoopEmbeddedTaskExecutor(),
- // TODO: IGNITE-404: Uncomment when fixed.
- //cfg.isExternalExecution() ? new HadoopExternalTaskExecutor() : new HadoopEmbeddedTaskExecutor(),
- new HadoopShuffle());
+ HadoopClassLoader.hadoopUrls();
+ hctx = new HadoopContext(
+ ctx,
+ cfg,
+ new HadoopJobTracker(),
+ new HadoopEmbeddedTaskExecutor(),
+ // TODO: IGNITE-404: Uncomment when fixed.
+ //cfg.isExternalExecution() ? new HadoopExternalTaskExecutor() : new HadoopEmbeddedTaskExecutor(),
+ new HadoopShuffle());
- for (HadoopComponent c : hctx.components())
- c.start(hctx);
+ for (HadoopComponent c : hctx.components())
+ c.start(hctx);
- hadoop = new HadoopImpl(this);
- }
+ hadoop = new HadoopImpl(this);
}
/** {@inheritDoc} */
[14/20] ignite git commit: Merge remote-tracking branch
'upstream/gridgain-7.5.28' into gridgain-7.5.28
Posted by vo...@apache.org.
Merge remote-tracking branch 'upstream/gridgain-7.5.28' into gridgain-7.5.28
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6664ce8c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6664ce8c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6664ce8c
Branch: refs/heads/ignite-3410
Commit: 6664ce8cc5f08d90ea40cb494e4e06f06badc167
Parents: d146406 6bf57cc
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Wed Jun 29 16:52:00 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Jun 29 16:52:00 2016 +0300
----------------------------------------------------------------------
.../config/benchmark-cache-load-win.properties | 60 +
.../config/benchmark-cache-load.properties | 92 ++
.../yardstick/config/benchmark-full.properties | 116 ++
.../config/ignite-base-load-config.xml | 255 ++++
.../config/ignite-cache-load-config.xml | 71 ++
modules/yardstick/config/queries.sql | 2 +
.../yardstick/IgniteBenchmarkArguments.java | 50 +-
.../org/apache/ignite/yardstick/IgniteNode.java | 2 +
.../cache/IgniteCacheAbstractBenchmark.java | 59 +
.../yardstick/cache/IgniteGetTxBenchmark.java | 30 +
.../yardstick/cache/IgniteInvokeBenchmark.java | 65 +
.../cache/IgniteInvokeTxBenchmark.java | 30 +
.../IgniteCacheRandomOperationBenchmark.java | 1160 ++++++++++++++++++
.../yardstick/cache/load/model/ModelUtil.java | 181 +++
.../cache/load/model/key/Identifier.java | 113 ++
.../yardstick/cache/load/model/key/Mark.java | 115 ++
.../yardstick/cache/load/model/value/Car.java | 126 ++
.../yardstick/cache/load/model/value/Color.java | 50 +
.../yardstick/cache/load/model/value/Truck.java | 69 ++
.../ignite/yardstick/cache/model/Account.java | 6 +
.../ignite/yardstick/cache/model/Person1.java | 2 +-
.../ignite/yardstick/cache/model/Person2.java | 2 +-
.../ignite/yardstick/cache/model/SampleKey.java | 2 +-
.../yardstick/cache/model/SampleValue.java | 2 +-
24 files changed, 2655 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[17/20] ignite git commit: Revert "IGNITE-3154: Fix for optimized
BinaryType field lookup."
Posted by vo...@apache.org.
Revert "IGNITE-3154: Fix for optimized BinaryType field lookup."
This reverts commit 1873fd8196a3d44482575a78dea8da86461e0ba4.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/02d395cf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/02d395cf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/02d395cf
Branch: refs/heads/ignite-3410
Commit: 02d395cffb63bf5b446d26440cc129fa90381824
Parents: 6664ce8
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 30 09:26:44 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 30 09:26:44 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/binary/BinaryEnumObjectImpl.java | 7 +------
.../apache/ignite/internal/binary/BinaryObjectEx.java | 11 -----------
.../ignite/internal/binary/BinaryObjectExImpl.java | 4 ++--
.../ignite/internal/binary/BinaryObjectImpl.java | 5 -----
.../internal/binary/BinaryObjectOffheapImpl.java | 5 -----
.../ignite/internal/binary/BinaryTypeProxy.java | 6 +-----
.../apache/ignite/internal/binary/BinaryUtils.java | 14 --------------
.../cache/binary/CacheObjectBinaryProcessorImpl.java | 7 +++----
.../internal/processors/query/GridQueryProcessor.java | 11 +++++------
9 files changed, 12 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/02d395cf/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
index 8cdd4d8..eea85ef9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
@@ -135,11 +135,6 @@ public class BinaryEnumObjectImpl implements BinaryObjectEx, Externalizable, Cac
}
/** {@inheritDoc} */
- @Nullable @Override public BinaryType rawType() throws BinaryObjectException {
- return BinaryUtils.type(ctx, this);
- }
-
- /** {@inheritDoc} */
@Override public <F> F field(String fieldName) throws BinaryObjectException {
return null;
}
@@ -204,7 +199,7 @@ public class BinaryEnumObjectImpl implements BinaryObjectEx, Externalizable, Cac
BinaryType type;
try {
- type = rawType();
+ type = type();
}
catch (Exception e) {
type = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/02d395cf/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java
index e3566bc..ba15d13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java
@@ -18,9 +18,6 @@
package org.apache.ignite.internal.binary;
import org.apache.ignite.binary.BinaryObject;
-import org.apache.ignite.binary.BinaryObjectException;
-import org.apache.ignite.binary.BinaryType;
-import org.jetbrains.annotations.Nullable;
/**
* Extended binary object interface.
@@ -30,12 +27,4 @@ public interface BinaryObjectEx extends BinaryObject {
* @return Type ID.
*/
public int typeId();
-
- /**
- * Get raw type.
- *
- * @return Raw type
- * @throws BinaryObjectException If failed.
- */
- @Nullable public BinaryType rawType() throws BinaryObjectException;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/02d395cf/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
index b4e909e..ddf17f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
@@ -176,7 +176,7 @@ public abstract class BinaryObjectExImpl implements BinaryObjectEx {
BinaryType meta;
try {
- meta = rawType();
+ meta = type();
}
catch (BinaryObjectException ignore) {
meta = null;
@@ -222,7 +222,7 @@ public abstract class BinaryObjectExImpl implements BinaryObjectEx {
Integer idHash0 = handles.get(val);
if (idHash0 != null) { // Circular reference.
- BinaryType meta0 = po.rawType();
+ BinaryType meta0 = po.type();
assert meta0 != null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/02d395cf/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
index 468cc25..9ebee0b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
@@ -231,11 +231,6 @@ public final class BinaryObjectImpl extends BinaryObjectExImpl implements Extern
}
/** {@inheritDoc} */
- @Nullable @Override public BinaryType rawType() throws BinaryObjectException {
- return BinaryUtils.type(ctx, this);
- }
-
- /** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Nullable @Override public <F> F field(String fieldName) throws BinaryObjectException {
return (F) reader(null).unmarshalField(fieldName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/02d395cf/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
index 5e9ee95..60527d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
@@ -146,11 +146,6 @@ public class BinaryObjectOffheapImpl extends BinaryObjectExImpl implements Exter
}
/** {@inheritDoc} */
- @Nullable @Override public BinaryType rawType() throws BinaryObjectException {
- return BinaryUtils.type(ctx, this);
- }
-
- /** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Nullable @Override public <F> F field(String fieldName) throws BinaryObjectException {
return (F) reader(null).unmarshalField(fieldName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/02d395cf/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
index 17b0bc6..e10f2f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.binary;
import org.apache.ignite.binary.BinaryField;
-import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.binary.BinaryType;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -87,16 +86,13 @@ public class BinaryTypeProxy implements BinaryType {
}
/**
- * @return Target type.
+ * @return Target ype.
*/
private BinaryType target() {
if (target == null) {
synchronized (this) {
if (target == null) {
target = ctx.metadata(typeId);
-
- if (target == null)
- throw new BinaryObjectException("Failed to get binary type details [typeId=" + typeId + ']');
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/02d395cf/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index b7ab310..e986228 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -2200,20 +2200,6 @@ public class BinaryUtils {
}
/**
- * Create binary type.
- *
- * @param ctx Context.
- * @param obj Binary object.
- * @return Binary type.
- */
- public static BinaryType type(BinaryContext ctx, BinaryObjectEx obj) {
- if (ctx == null)
- throw new BinaryObjectException("BinaryContext is not set for the object.");
-
- return ctx.metadata(obj.typeId());
- }
-
- /**
* Enum type.
*/
private static class EnumType {
http://git-wip-us.apache.org/repos/asf/ignite/blob/02d395cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 18364f0..d5756a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import javax.cache.Cache;
@@ -670,9 +671,7 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
*/
public Object affinityKey(BinaryObject po) {
try {
- assert po instanceof BinaryObjectEx;
-
- BinaryType meta = ((BinaryObjectEx)po).rawType();
+ BinaryType meta = po.type();
if (meta != null) {
String affKeyFieldName = meta.affinityKeyFieldName();
@@ -680,7 +679,7 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
if (affKeyFieldName != null)
return po.field(affKeyFieldName);
}
- else {
+ else if (po instanceof BinaryObjectEx) {
int id = ((BinaryObjectEx)po).typeId();
String affKeyFieldName = binaryCtx.affinityKeyFieldName(id);
http://git-wip-us.apache.org/repos/asf/ignite/blob/02d395cf/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index c4c3ad6..074e7e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -36,7 +36,6 @@ import org.apache.ignite.events.CacheQueryExecutedEvent;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.binary.BinaryMarshaller;
-import org.apache.ignite.internal.binary.BinaryObjectEx;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
import org.apache.ignite.internal.processors.cache.CacheObject;
@@ -2005,9 +2004,9 @@ public class GridQueryProcessor extends GridProcessorAdapter {
obj = isKeyProp0 == 1 ? key : val;
}
- assert obj instanceof BinaryObjectEx;
+ assert obj instanceof BinaryObject;
- BinaryObjectEx obj0 = (BinaryObjectEx)obj;
+ BinaryObject obj0 = (BinaryObject)obj;
return fieldValue(obj0);
}
@@ -2018,11 +2017,11 @@ public class GridQueryProcessor extends GridProcessorAdapter {
* @param obj Target object.
* @return Binary field.
*/
- private BinaryField binaryField(BinaryObjectEx obj) {
+ private BinaryField binaryField(BinaryObject obj) {
BinaryField field0 = field;
if (field0 == null && !fieldTaken) {
- BinaryType type = obj.rawType();
+ BinaryType type = obj.type();
if (type != null) {
field0 = type.field(propName);
@@ -2045,7 +2044,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
* @return Field value.
*/
@SuppressWarnings("IfMayBeConditional")
- private Object fieldValue(BinaryObjectEx obj) {
+ private Object fieldValue(BinaryObject obj) {
BinaryField field = binaryField(obj);
if (field != null)
[16/20] ignite git commit: Fixed failing test.
Posted by vo...@apache.org.
Fixed failing test.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7086cc02
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7086cc02
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7086cc02
Branch: refs/heads/ignite-3410
Commit: 7086cc0258106120f4a476c9b783b40904675d04
Parents: 6fd69af
Author: sboikov <sb...@gridgain.com>
Authored: Thu Jun 30 09:24:46 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Jun 30 09:24:46 2016 +0300
----------------------------------------------------------------------
.../processors/query/h2/sql/GridQueryParsingTest.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7086cc02/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
index ce1f241..9fb5765 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
@@ -292,7 +292,10 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
* @param sql2 Sql 2.
*/
private void assertSqlEquals(String sql1, String sql2) {
- assertEquals(normalizeSql(sql1), normalizeSql(sql2));
+ String nsql1 = normalizeSql(sql1);
+ String nsql2 = normalizeSql(sql2);
+
+ assertEquals(nsql1, nsql2);
}
/**
@@ -301,7 +304,7 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
private static String normalizeSql(String sql) {
return sql.toLowerCase()
.replaceAll("/\\*(?:.|\r|\n)*?\\*/", " ")
- .replaceAll("\\s*on\\s+1\\s*=\\s*1\\s*", " ")
+ .replaceAll("\\s*on\\s+1\\s*=\\s*1\\s*", " on true ")
.replaceAll("\\s+", " ")
.replaceAll("\\( +", "(")
.replaceAll(" +\\)", ")")
@@ -314,9 +317,9 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
private void checkQuery(String qry) throws Exception {
Prepared prepared = parse(qry);
- GridSqlQueryParser ses = new GridSqlQueryParser();
+ GridSqlQuery gQry = new GridSqlQueryParser().parse(prepared);
- String res = ses.parse(prepared).getSQL();
+ String res = gQry.getSQL();
System.out.println(normalizeSql(res));
[13/20] ignite git commit: IGNITE-3383: Hadoop: fixed missing Hadoop
environment printout.
Posted by vo...@apache.org.
IGNITE-3383: Hadoop: fixed missing Hadoop environment printout.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d146406f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d146406f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d146406f
Branch: refs/heads/ignite-3410
Commit: d146406fa731373dd1b943fd8db18762d0e78fed
Parents: 1873fd8
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Wed Jun 29 16:51:23 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Jun 29 16:51:23 2016 +0300
----------------------------------------------------------------------
.../src/main/java/org/apache/ignite/internal/IgniteKernal.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d146406f/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index b6ba4cf..9d42598 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1097,7 +1097,11 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
throw new IgniteCheckedException("Hadoop module cannot be used with peer class loading enabled " +
"(set IgniteConfiguration.peerClassLoadingEnabled to \"false\").");
- return IgniteComponentType.HADOOP.createIfInClassPath(ctx, true);
+ HadoopProcessorAdapter res = IgniteComponentType.HADOOP.createIfInClassPath(ctx, true);
+
+ res.validateEnvironment();
+
+ return res;
}
else {
HadoopProcessorAdapter cmp = null;
[05/20] ignite git commit: IGNITE-2115: Backported from master to
7.5.28.
Posted by vo...@apache.org.
IGNITE-2115: Backported from master to 7.5.28.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a77e6901
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a77e6901
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a77e6901
Branch: refs/heads/ignite-3410
Commit: a77e6901a4e9e9f1116698fc65e6a2ffbe68f08f
Parents: 62c7143
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jun 28 14:42:12 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jun 28 14:42:12 2016 +0300
----------------------------------------------------------------------
examples/config/filesystem/example-igfs.xml | 3 --
.../apache/ignite/internal/IgniteKernal.java | 31 ++++++++++++++++++--
modules/core/src/test/config/igfs-loopback.xml | 7 -----
modules/core/src/test/config/igfs-shmem.xml | 7 -----
4 files changed, 28 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a77e6901/examples/config/filesystem/example-igfs.xml
----------------------------------------------------------------------
diff --git a/examples/config/filesystem/example-igfs.xml b/examples/config/filesystem/example-igfs.xml
index 0a0b1b4..a72ddce 100644
--- a/examples/config/filesystem/example-igfs.xml
+++ b/examples/config/filesystem/example-igfs.xml
@@ -50,9 +50,6 @@
Configuration below demonstrates how to setup a IgniteFs node with file data.
-->
<bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
- <!-- Set to true to enable cluster-aware class loading for examples, default is false. -->
- <property name="peerClassLoadingEnabled" value="true"/>
-
<property name="marshaller">
<bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
<!-- Set to false to allow non-serializable objects in examples, default is true. -->
http://git-wip-us.apache.org/repos/asf/ignite/blob/a77e6901/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index d1f3ef5..3572293 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -114,6 +114,7 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
import org.apache.ignite.internal.processors.hadoop.Hadoop;
+import org.apache.ignite.internal.processors.hadoop.HadoopNoopProcessor;
import org.apache.ignite.internal.processors.job.GridJobProcessor;
import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor;
@@ -862,9 +863,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
startProcessor(new DataStreamProcessor(ctx));
startProcessor((GridProcessor)IGFS.create(ctx, F.isEmpty(cfg.getFileSystemConfiguration())));
startProcessor(new GridContinuousProcessor(ctx));
- startProcessor((GridProcessor)(cfg.isPeerClassLoadingEnabled() ?
- IgniteComponentType.HADOOP.create(ctx, true): // No-op when peer class loading is enabled.
- IgniteComponentType.HADOOP.createIfInClassPath(ctx, cfg.getHadoopConfiguration() != null)));
+ startProcessor((GridProcessor)createHadoopComponent());
startProcessor(new DataStructuresProcessor(ctx));
startProcessor(createComponent(PlatformProcessor.class, ctx));
@@ -1085,6 +1084,32 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/**
+ * Create Hadoop component.
+ *
+ * @return Non-null Hadoop component: workable or no-op.
+ * @throws IgniteCheckedException if the component is mandatory and cannot be initialized.
+ */
+ private GridComponent createHadoopComponent() throws IgniteCheckedException {
+ GridComponent cmp;
+
+ if (cfg.isPeerClassLoadingEnabled()) {
+ cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false);
+
+ if (!(cmp instanceof HadoopNoopProcessor)) {
+ U.warn(log, "Hadoop module is found in classpath, but it will not be started because peer class " +
+ "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled to \"false\" to start " +
+ "Hadoop module).");
+
+ cmp = IgniteComponentType.HADOOP.create(ctx, true/*no-op*/);
+ }
+ }
+ else
+ cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, cfg.getHadoopConfiguration() != null);
+
+ return cmp;
+ }
+
+ /**
* Validates common configuration parameters.
*
* @param cfg Configuration.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a77e6901/modules/core/src/test/config/igfs-loopback.xml
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/igfs-loopback.xml b/modules/core/src/test/config/igfs-loopback.xml
index c01fb98..dbbfb4e 100644
--- a/modules/core/src/test/config/igfs-loopback.xml
+++ b/modules/core/src/test/config/igfs-loopback.xml
@@ -54,13 +54,6 @@
-->
<bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
<!--
- For better performance set this property to false in case
- peer deployment is not used.
- Default value is false.
- -->
- <property name="peerClassLoadingEnabled" value="true"/>
-
- <!--
Configure optimized marshaller.
-->
<property name="marshaller">
http://git-wip-us.apache.org/repos/asf/ignite/blob/a77e6901/modules/core/src/test/config/igfs-shmem.xml
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/igfs-shmem.xml b/modules/core/src/test/config/igfs-shmem.xml
index b38544f..c4c8704 100644
--- a/modules/core/src/test/config/igfs-shmem.xml
+++ b/modules/core/src/test/config/igfs-shmem.xml
@@ -54,13 +54,6 @@
-->
<bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
<!--
- For better performance set this property to false in case
- peer deployment is not used.
- Default value is false.
- -->
- <property name="peerClassLoadingEnabled" value="true"/>
-
- <!--
Configure optimized marshaller.
-->
<property name="marshaller">
[20/20] ignite git commit: IGNITE-3184: Hadoop: fixed Windows
setenv.bat.
Posted by vo...@apache.org.
IGNITE-3184: Hadoop: fixed Windows setenv.bat.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/17276475
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/17276475
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/17276475
Branch: refs/heads/ignite-3410
Commit: 17276475714cf2bb794384b02132d1e7be4fe547
Parents: 4113d0f
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 30 16:15:15 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 30 16:15:15 2016 +0300
----------------------------------------------------------------------
bin/include/setenv.bat | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/17276475/bin/include/setenv.bat
----------------------------------------------------------------------
diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat
index dec1ee5..9d55521 100644
--- a/bin/include/setenv.bat
+++ b/bin/include/setenv.bat
@@ -47,9 +47,9 @@ if exist %IGNITE_HOME%\libs\ignite-hadoop set HADOOP_EDITION=1
if defined USER_LIBS set IGNITE_LIBS=%USER_LIBS%;%IGNITE_LIBS%
-FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\ignite-hadoop\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i
+if "%HADOOP_EDITION%" == "1" FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\ignite-hadoop\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i
-if "[%IGNITE_HADOOP_CLASSPATH%]" == "[]" exit 1
+if "%IGNITE_HADOOP_CLASSPATH%" == "" goto :eof
set IGNITE_LIBS=%IGNITE_LIBS%;%IGNITE_HADOOP_CLASSPATH%
[09/20] ignite git commit: IGNITE-3154: Fix for optimized BinaryType
field lookup.
Posted by vo...@apache.org.
IGNITE-3154: Fix for optimized BinaryType field lookup.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1873fd81
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1873fd81
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1873fd81
Branch: refs/heads/ignite-3410
Commit: 1873fd8196a3d44482575a78dea8da86461e0ba4
Parents: 591a953
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Wed Jun 29 15:22:54 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Jun 29 15:22:54 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/binary/BinaryEnumObjectImpl.java | 7 ++++++-
.../apache/ignite/internal/binary/BinaryObjectEx.java | 11 +++++++++++
.../ignite/internal/binary/BinaryObjectExImpl.java | 4 ++--
.../ignite/internal/binary/BinaryObjectImpl.java | 5 +++++
.../internal/binary/BinaryObjectOffheapImpl.java | 5 +++++
.../ignite/internal/binary/BinaryTypeProxy.java | 6 +++++-
.../apache/ignite/internal/binary/BinaryUtils.java | 14 ++++++++++++++
.../cache/binary/CacheObjectBinaryProcessorImpl.java | 7 ++++---
.../internal/processors/query/GridQueryProcessor.java | 11 ++++++-----
9 files changed, 58 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1873fd81/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
index eea85ef9..8cdd4d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
@@ -135,6 +135,11 @@ public class BinaryEnumObjectImpl implements BinaryObjectEx, Externalizable, Cac
}
/** {@inheritDoc} */
+ @Nullable @Override public BinaryType rawType() throws BinaryObjectException {
+ return BinaryUtils.type(ctx, this);
+ }
+
+ /** {@inheritDoc} */
@Override public <F> F field(String fieldName) throws BinaryObjectException {
return null;
}
@@ -199,7 +204,7 @@ public class BinaryEnumObjectImpl implements BinaryObjectEx, Externalizable, Cac
BinaryType type;
try {
- type = type();
+ type = rawType();
}
catch (Exception e) {
type = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1873fd81/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java
index ba15d13..e3566bc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java
@@ -18,6 +18,9 @@
package org.apache.ignite.internal.binary;
import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryType;
+import org.jetbrains.annotations.Nullable;
/**
* Extended binary object interface.
@@ -27,4 +30,12 @@ public interface BinaryObjectEx extends BinaryObject {
* @return Type ID.
*/
public int typeId();
+
+ /**
+ * Get raw type.
+ *
+ * @return Raw type
+ * @throws BinaryObjectException If failed.
+ */
+ @Nullable public BinaryType rawType() throws BinaryObjectException;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1873fd81/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
index ddf17f0..b4e909e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java
@@ -176,7 +176,7 @@ public abstract class BinaryObjectExImpl implements BinaryObjectEx {
BinaryType meta;
try {
- meta = type();
+ meta = rawType();
}
catch (BinaryObjectException ignore) {
meta = null;
@@ -222,7 +222,7 @@ public abstract class BinaryObjectExImpl implements BinaryObjectEx {
Integer idHash0 = handles.get(val);
if (idHash0 != null) { // Circular reference.
- BinaryType meta0 = po.type();
+ BinaryType meta0 = po.rawType();
assert meta0 != null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1873fd81/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
index 9ebee0b..468cc25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
@@ -231,6 +231,11 @@ public final class BinaryObjectImpl extends BinaryObjectExImpl implements Extern
}
/** {@inheritDoc} */
+ @Nullable @Override public BinaryType rawType() throws BinaryObjectException {
+ return BinaryUtils.type(ctx, this);
+ }
+
+ /** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Nullable @Override public <F> F field(String fieldName) throws BinaryObjectException {
return (F) reader(null).unmarshalField(fieldName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1873fd81/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
index 60527d1..5e9ee95 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
@@ -146,6 +146,11 @@ public class BinaryObjectOffheapImpl extends BinaryObjectExImpl implements Exter
}
/** {@inheritDoc} */
+ @Nullable @Override public BinaryType rawType() throws BinaryObjectException {
+ return BinaryUtils.type(ctx, this);
+ }
+
+ /** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Nullable @Override public <F> F field(String fieldName) throws BinaryObjectException {
return (F) reader(null).unmarshalField(fieldName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1873fd81/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
index e10f2f7..17b0bc6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.binary;
import org.apache.ignite.binary.BinaryField;
+import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.binary.BinaryType;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -86,13 +87,16 @@ public class BinaryTypeProxy implements BinaryType {
}
/**
- * @return Target ype.
+ * @return Target type.
*/
private BinaryType target() {
if (target == null) {
synchronized (this) {
if (target == null) {
target = ctx.metadata(typeId);
+
+ if (target == null)
+ throw new BinaryObjectException("Failed to get binary type details [typeId=" + typeId + ']');
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1873fd81/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index e986228..b7ab310 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -2200,6 +2200,20 @@ public class BinaryUtils {
}
/**
+ * Create binary type.
+ *
+ * @param ctx Context.
+ * @param obj Binary object.
+ * @return Binary type.
+ */
+ public static BinaryType type(BinaryContext ctx, BinaryObjectEx obj) {
+ if (ctx == null)
+ throw new BinaryObjectException("BinaryContext is not set for the object.");
+
+ return ctx.metadata(obj.typeId());
+ }
+
+ /**
* Enum type.
*/
private static class EnumType {
http://git-wip-us.apache.org/repos/asf/ignite/blob/1873fd81/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index d5756a3..18364f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -26,7 +26,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import javax.cache.Cache;
@@ -671,7 +670,9 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
*/
public Object affinityKey(BinaryObject po) {
try {
- BinaryType meta = po.type();
+ assert po instanceof BinaryObjectEx;
+
+ BinaryType meta = ((BinaryObjectEx)po).rawType();
if (meta != null) {
String affKeyFieldName = meta.affinityKeyFieldName();
@@ -679,7 +680,7 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
if (affKeyFieldName != null)
return po.field(affKeyFieldName);
}
- else if (po instanceof BinaryObjectEx) {
+ else {
int id = ((BinaryObjectEx)po).typeId();
String affKeyFieldName = binaryCtx.affinityKeyFieldName(id);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1873fd81/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 074e7e5..c4c3ad6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -36,6 +36,7 @@ import org.apache.ignite.events.CacheQueryExecutedEvent;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.internal.binary.BinaryObjectEx;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
import org.apache.ignite.internal.processors.cache.CacheObject;
@@ -2004,9 +2005,9 @@ public class GridQueryProcessor extends GridProcessorAdapter {
obj = isKeyProp0 == 1 ? key : val;
}
- assert obj instanceof BinaryObject;
+ assert obj instanceof BinaryObjectEx;
- BinaryObject obj0 = (BinaryObject)obj;
+ BinaryObjectEx obj0 = (BinaryObjectEx)obj;
return fieldValue(obj0);
}
@@ -2017,11 +2018,11 @@ public class GridQueryProcessor extends GridProcessorAdapter {
* @param obj Target object.
* @return Binary field.
*/
- private BinaryField binaryField(BinaryObject obj) {
+ private BinaryField binaryField(BinaryObjectEx obj) {
BinaryField field0 = field;
if (field0 == null && !fieldTaken) {
- BinaryType type = obj.type();
+ BinaryType type = obj.rawType();
if (type != null) {
field0 = type.field(propName);
@@ -2044,7 +2045,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
* @return Field value.
*/
@SuppressWarnings("IfMayBeConditional")
- private Object fieldValue(BinaryObject obj) {
+ private Object fieldValue(BinaryObjectEx obj) {
BinaryField field = binaryField(obj);
if (field != null)
[04/20] ignite git commit: IGNITE-3154: Optimized BinaryField lookup
within a BinaryType.
Posted by vo...@apache.org.
IGNITE-3154: Optimized BinaryField lookup within a BinaryType.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/62c71434
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/62c71434
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/62c71434
Branch: refs/heads/ignite-3410
Commit: 62c714340ef6c40de4dcdb6d9ecaf11ab30df834
Parents: ac092f4
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jun 28 11:37:21 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jun 28 11:37:21 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/binary/BinaryContext.java | 1 -
.../internal/binary/BinaryEnumObjectImpl.java | 2 +-
.../internal/binary/BinaryObjectImpl.java | 5 +-
.../binary/BinaryObjectOffheapImpl.java | 5 +-
.../ignite/internal/binary/BinaryTypeProxy.java | 107 +++++++++++++++++++
.../ignite/internal/binary/BinaryUtils.java | 21 +++-
6 files changed, 129 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/62c71434/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index d39f9c6..3853841 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -83,7 +83,6 @@ import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaFileUnlockProcess
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaUpdatePropertiesProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaUpdateTimesProcessor;
import org.apache.ignite.internal.processors.platform.PlatformJavaObjectFactoryProxy;
-import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.lang.GridMapEntry;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T2;
http://git-wip-us.apache.org/repos/asf/ignite/blob/62c71434/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
index 60642ab..eea85ef9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
@@ -131,7 +131,7 @@ public class BinaryEnumObjectImpl implements BinaryObjectEx, Externalizable, Cac
/** {@inheritDoc} */
@Override public BinaryType type() throws BinaryObjectException {
- return ctx.metadata(typeId());
+ return BinaryUtils.typeProxy(ctx, this);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/62c71434/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
index 173bb6e..9ebee0b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
@@ -227,10 +227,7 @@ public final class BinaryObjectImpl extends BinaryObjectExImpl implements Extern
/** {@inheritDoc} */
@Nullable @Override public BinaryType type() throws BinaryObjectException {
- if (ctx == null)
- throw new BinaryObjectException("BinaryContext is not set for the object.");
-
- return ctx.metadata(typeId());
+ return BinaryUtils.typeProxy(ctx, this);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/62c71434/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
index 509247f..60527d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
@@ -142,10 +142,7 @@ public class BinaryObjectOffheapImpl extends BinaryObjectExImpl implements Exter
/** {@inheritDoc} */
@Nullable @Override public BinaryType type() throws BinaryObjectException {
- if (ctx == null)
- throw new BinaryObjectException("BinaryContext is not set for the object.");
-
- return ctx.metadata(typeId());
+ return BinaryUtils.typeProxy(ctx, this);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/62c71434/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
new file mode 100644
index 0000000..e10f2f7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import org.apache.ignite.binary.BinaryField;
+import org.apache.ignite.binary.BinaryType;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+import java.util.Collection;
+
+/**
+ * Binary type proxy. Is used to delay or completely avoid metadata lookup.
+ */
+public class BinaryTypeProxy implements BinaryType {
+ /** Binary context. */
+ @GridToStringExclude
+ private final BinaryContext ctx;
+
+ /** Type ID. */
+ private final int typeId;
+
+ /** Target type. */
+ @GridToStringExclude
+ private volatile BinaryType target;
+
+ /**
+ * Constrcutor.
+ *
+ * @param ctx Context.
+ * @param typeId Type ID.
+ */
+ public BinaryTypeProxy(BinaryContext ctx, int typeId) {
+ this.ctx = ctx;
+ this.typeId = typeId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int typeId() {
+ return typeId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public BinaryField field(String fieldName) {
+ return ctx.createField(typeId, fieldName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String typeName() {
+ return target().typeName();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<String> fieldNames() {
+ return target().fieldNames();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String fieldTypeName(String fieldName) {
+ return target().fieldTypeName(fieldName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String affinityKeyFieldName() {
+ return target().affinityKeyFieldName();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isEnum() {
+ return target().isEnum();
+ }
+
+ /**
+ * @return Target ype.
+ */
+ private BinaryType target() {
+ if (target == null) {
+ synchronized (this) {
+ if (target == null) {
+ target = ctx.metadata(typeId);
+ }
+ }
+ }
+
+ return target;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(BinaryTypeProxy.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/62c71434/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 0250e3b..e986228 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -54,6 +54,7 @@ import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.binary.BinaryRawReader;
import org.apache.ignite.binary.BinaryRawWriter;
+import org.apache.ignite.binary.BinaryType;
import org.apache.ignite.binary.Binarylizable;
import org.apache.ignite.internal.binary.builder.BinaryLazyValue;
import org.apache.ignite.internal.binary.streams.BinaryInputStream;
@@ -602,6 +603,7 @@ public class BinaryUtils {
* @param map Map.
* @return New map of the same type or null.
*/
+ @SuppressWarnings("unchecked")
public static <K, V> Map<K, V> newKnownMap(Object map) {
Class<?> cls = map == null ? null : map.getClass();
@@ -676,6 +678,7 @@ public class BinaryUtils {
* @param col Collection.
* @return New empty collection.
*/
+ @SuppressWarnings("unchecked")
public static <V> Collection<V> newKnownCollection(Object col) {
Class<?> cls = col == null ? null : col.getClass();
@@ -1392,6 +1395,7 @@ public class BinaryUtils {
/**
* @return Value.
*/
+ @SuppressWarnings("ConstantConditions")
public static Object doReadProxy(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr,
BinaryReaderHandlesHolder handles) {
Class<?>[] intfs = new Class<?>[in.readInt()];
@@ -2123,8 +2127,7 @@ public class BinaryUtils {
throw new BinaryObjectException("Malformed input around byte: " + (off - 1));
res[charArrCnt++] = (char)(((c & 0x0F) << 12) |
- ((c2 & 0x3F) << 6) |
- ((c3 & 0x3F) << 0));
+ ((c2 & 0x3F) << 6) | (c3 & 0x3F));
break;
default:
@@ -2183,6 +2186,20 @@ public class BinaryUtils {
}
/**
+ * Create binary type proxy.
+ *
+ * @param ctx Context.
+ * @param obj Binary object.
+ * @return Binary type proxy.
+ */
+ public static BinaryType typeProxy(BinaryContext ctx, BinaryObjectEx obj) {
+ if (ctx == null)
+ throw new BinaryObjectException("BinaryContext is not set for the object.");
+
+ return new BinaryTypeProxy(ctx, obj.typeId());
+ }
+
+ /**
* Enum type.
*/
private static class EnumType {
[07/20] ignite git commit: updated BinaryObjectBuilder documentation
Posted by vo...@apache.org.
updated BinaryObjectBuilder documentation
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7f0b9ea4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7f0b9ea4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7f0b9ea4
Branch: refs/heads/ignite-3410
Commit: 7f0b9ea4190d84b67165aca75ed8c0ff91127903
Parents: ac092f4
Author: Denis Magda <dm...@gridgain.com>
Authored: Wed Jun 29 14:38:44 2016 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Wed Jun 29 14:38:44 2016 +0300
----------------------------------------------------------------------
.../org/apache/ignite/binary/BinaryObjectBuilder.java | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7f0b9ea4/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectBuilder.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectBuilder.java b/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectBuilder.java
index 9fd8420..5c90763 100644
--- a/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectBuilder.java
@@ -60,6 +60,17 @@ import org.jetbrains.annotations.Nullable;
* String city = personBinaryObj.getField("address").getField("city");
* </pre>
*
+ * <p>
+ * Make sure to set values for all the fields that an object from your domain model has.
+ * If you need to set null as a value use {@link #setField(String, Object, Class)} method directly specifying
+ * field's type.
+ * <p>
+ * If to follow this recommendation you'll reduce the size of internal metadata object that
+ * every binary object of a particular type has. Usually the metadata size grows because particular fields are not set
+ * to an instance of a binary object constructed with the builder. Every time when you construct an object
+ * setting only a subset of the fields the metadata object related to this type is expanded by the metadata
+ * processor which treats every new combination of the fields as the new version of the binary object.
+ *
* @see org.apache.ignite.IgniteBinary#builder(String)
* @see org.apache.ignite.IgniteBinary#builder(BinaryObject)
*/
[15/20] ignite git commit: Yardstick backport from master rat fix
Posted by vo...@apache.org.
Yardstick backport from master rat fix
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6fd69af9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6fd69af9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6fd69af9
Branch: refs/heads/ignite-3410
Commit: 6fd69af95050b7490ed31f0abc73bf3e278c7522
Parents: 6664ce8
Author: Anton Vinogradov <av...@apache.org>
Authored: Wed Jun 29 18:09:37 2016 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed Jun 29 18:09:37 2016 +0300
----------------------------------------------------------------------
parent/pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6fd69af9/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 8aa0da7..1553612 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -768,6 +768,7 @@
<exclude>**/*.gar</exclude><!--bin-files-->
<exclude>**/licenses/*.txt</exclude><!--files of licenses-->
<exclude>**/*readme*.txt</exclude><!--readme files-->
+ <exclude>**/*.sql</exclude><!--sql files-->
<exclude>**/*README*.txt</exclude><!--readme files-->
<exclude>**/*README*.md</exclude><!--readme files-->
<exclude>**/*.timestamp</exclude><!--tmp-files-->
[08/20] ignite git commit: Merge remote-tracking branch
'community/gridgain-7.5.28' into gridgain-7.5.28
Posted by vo...@apache.org.
Merge remote-tracking branch 'community/gridgain-7.5.28' into gridgain-7.5.28
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/591a9535
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/591a9535
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/591a9535
Branch: refs/heads/ignite-3410
Commit: 591a9535e8279f2af0ccf644696fcc736dce2b77
Parents: 7f0b9ea bdeef4c
Author: Denis Magda <dm...@gridgain.com>
Authored: Wed Jun 29 14:39:02 2016 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Wed Jun 29 14:39:02 2016 +0300
----------------------------------------------------------------------
examples/config/filesystem/example-igfs.xml | 3 -
.../apache/ignite/internal/IgniteKernal.java | 49 ++++++++-
.../ignite/internal/binary/BinaryContext.java | 1 -
.../internal/binary/BinaryEnumObjectImpl.java | 2 +-
.../internal/binary/BinaryObjectImpl.java | 5 +-
.../binary/BinaryObjectOffheapImpl.java | 5 +-
.../ignite/internal/binary/BinaryTypeProxy.java | 107 +++++++++++++++++++
.../ignite/internal/binary/BinaryUtils.java | 21 +++-
.../processors/hadoop/HadoopNoopProcessor.java | 5 +
.../hadoop/HadoopProcessorAdapter.java | 7 ++
modules/core/src/test/config/igfs-loopback.xml | 7 --
modules/core/src/test/config/igfs-shmem.xml | 7 --
.../processors/hadoop/HadoopClassLoader.java | 2 +-
.../processors/hadoop/HadoopProcessor.java | 52 ++++-----
14 files changed, 210 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
[02/20] ignite git commit: ignite-3361 Fixed condition in
GridServiceProcessor.reassign.
Posted by vo...@apache.org.
ignite-3361 Fixed condition in GridServiceProcessor.reassign.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/76c76f89
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/76c76f89
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/76c76f89
Branch: refs/heads/ignite-3410
Commit: 76c76f89f82289ba30c340f96acfcdec2752be0d
Parents: 9007fb6
Author: sboikov <sb...@gridgain.com>
Authored: Mon Jun 27 15:53:03 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Jun 27 15:53:03 2016 +0300
----------------------------------------------------------------------
.../service/GridServiceProcessor.java | 2 +-
.../service/IgniteServiceReassignmentTest.java | 250 +++++++++++++++++++
.../testsuites/IgniteKernalSelfTestSuite.java | 2 +
3 files changed, 253 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/76c76f89/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index add90e2..53eaeb5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -942,7 +942,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
for (Map.Entry<UUID, Integer> e : entries) {
// Assign only the ones that have not been reused from previous assignments.
if (!used.contains(e.getKey())) {
- if (e.getValue() < maxPerNodeCnt) {
+ if (e.getValue() < maxPerNodeCnt || maxPerNodeCnt == 0) {
e.setValue(e.getValue() + 1);
if (--remainder == 0)
http://git-wip-us.apache.org/repos/asf/ignite/blob/76c76f89/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java
new file mode 100644
index 0000000..d7937ec
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgniteServiceReassignmentTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private ServiceConfiguration srvcCfg;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+ if (srvcCfg != null)
+ cfg.setServiceConfiguration(srvcCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNodeRestart1() throws Exception {
+ srvcCfg = serviceConfiguration();
+
+ Ignite node1 = startGrid(1);
+
+ assertEquals(42, serviceProxy(node1).foo());
+
+ srvcCfg = serviceConfiguration();
+
+ Ignite node2 = startGrid(2);
+
+ node1.close();
+
+ waitForService(node2);
+
+ assertEquals(42, serviceProxy(node2).foo());
+
+ srvcCfg = serviceConfiguration();
+
+ Ignite node3 = startGrid(3);
+
+ assertEquals(42, serviceProxy(node3).foo());
+
+ srvcCfg = serviceConfiguration();
+
+ node1 = startGrid(1);
+
+ assertEquals(42, serviceProxy(node1).foo());
+ assertEquals(42, serviceProxy(node2).foo());
+ assertEquals(42, serviceProxy(node3).foo());
+
+ node2.close();
+
+ waitForService(node1);
+
+ assertEquals(42, serviceProxy(node1).foo());
+ assertEquals(42, serviceProxy(node3).foo());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNodeRestart2() throws Exception {
+ startGrids(3);
+
+ ServiceConfiguration svcCfg = new ServiceConfiguration();
+
+ svcCfg.setName("DummyService");
+ svcCfg.setTotalCount(10);
+ svcCfg.setMaxPerNodeCount(1);
+ svcCfg.setService(new DummyService());
+
+ ignite(0).services().deploy(svcCfg);
+
+ for (int i = 0; i < 3; i++)
+ assertEquals(42, serviceProxy(ignite(i)).foo());
+
+ for (int i = 0; i < 3; i++)
+ startGrid(i + 3);
+
+ for (int i = 0; i < 3; i++)
+ stopGrid(i);
+
+ for (int i = 0; i < 3; i++)
+ assertEquals(42, serviceProxy(ignite(i + 3)).foo());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNodeRestartRandom() throws Exception {
+ final int NODES = 5;
+
+ Ignite ignite = startGridsMultiThreaded(NODES);
+
+ ignite.services().deploy(serviceConfiguration());
+
+ for (int i = 0; i < 30; i++) {
+ log.info("Iteration: " + i);
+
+ int stopIdx = ThreadLocalRandom.current().nextInt(NODES);
+
+ stopGrid(stopIdx);
+
+ for (int nodeIdx = 0; nodeIdx < NODES; nodeIdx++) {
+ if (nodeIdx == stopIdx)
+ continue;
+
+ waitForService(ignite(nodeIdx));
+
+ assertEquals(42, serviceProxy(ignite(nodeIdx)).foo());
+ }
+
+ startGrid(stopIdx);
+
+ for (int nodeIdx = 0; nodeIdx < NODES; nodeIdx++)
+ assertEquals(42, serviceProxy(ignite(nodeIdx)).foo());
+ }
+ }
+
+ /**
+ * @param node Node.
+ * @throws Exception If failed.
+ */
+ private void waitForService(final Ignite node) throws Exception {
+ assertTrue(GridTestUtils.waitForCondition(new PA() {
+ @Override public boolean apply() {
+ try {
+ serviceProxy(node).foo();
+
+ return true;
+ }
+ catch (IgniteException e) {
+ return false;
+ }
+ }
+ }, 5000));
+ }
+
+ /**
+ * @param node Node.
+ * @return Service proxy.
+ */
+ private static MyService serviceProxy(Ignite node) {
+ return node.services().serviceProxy("DummyService", MyService.class, true);
+ }
+
+ /**
+ * @return Service configuration.
+ */
+ private ServiceConfiguration serviceConfiguration() {
+ ServiceConfiguration svc = new ServiceConfiguration();
+
+ svc.setName("DummyService");
+ svc.setTotalCount(1);
+ svc.setService(new DummyService());
+
+ return svc;
+ }
+
+ /**
+ *
+ */
+ public interface MyService {
+ /**
+ * @return Dummy result.
+ */
+ int foo();
+ }
+
+ /**
+ *
+ */
+ static class DummyService implements MyService, Service {
+ /** */
+ @IgniteInstanceResource
+ private Ignite locNode;
+
+ /** {@inheritDoc} */
+ @Override public void cancel(ServiceContext ctx) {
+ locNode.log().info("Service cancelled [execId=" + ctx.executionId() +
+ ", node=" + locNode.cluster().localNode() + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public void init(ServiceContext ctx) {
+ locNode.log().info("Service initialized [execId=" + ctx.executionId() +
+ ", node=" + locNode.cluster().localNode() + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute(ServiceContext ctx) {
+ locNode.log().info("Service started [execId=" + ctx.executionId() +
+ ", node=" + locNode.cluster().localNode() + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public int foo() {
+ locNode.log().info("Service called.");
+
+ return 42;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/76c76f89/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index d990b32..471bea7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -65,6 +65,7 @@ import org.apache.ignite.internal.processors.service.IgniteServiceDeployment2Cla
import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingDefaultMarshallerTest;
import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingJdkMarshallerTest;
import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest;
+import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
import org.apache.ignite.internal.util.GridStartupWithSpecifiedWorkDirectorySelfTest;
import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
@@ -139,6 +140,7 @@ public class IgniteKernalSelfTestSuite extends TestSuite {
suite.addTestSuite(GridServicePackagePrivateSelfTest.class);
suite.addTestSuite(GridServiceSerializationSelfTest.class);
suite.addTestSuite(GridServiceProxyNodeStopSelfTest.class);
+ suite.addTestSuite(IgniteServiceReassignmentTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);
[19/20] ignite git commit: Merge remote-tracking branch
'upstream/gridgain-7.5.28' into gridgain-7.5.28
Posted by vo...@apache.org.
Merge remote-tracking branch 'upstream/gridgain-7.5.28' into gridgain-7.5.28
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4113d0ff
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4113d0ff
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4113d0ff
Branch: refs/heads/ignite-3410
Commit: 4113d0ffd1bb2c40320a589fe624716f6e386cdd
Parents: 6961e63 7086cc0
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 30 09:28:59 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 30 09:28:59 2016 +0300
----------------------------------------------------------------------
.../processors/query/h2/sql/GridQueryParsingTest.java | 11 +++++++----
parent/pom.xml | 1 +
2 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[03/20] ignite git commit: ignite-3372: IgniteDataStreamer:
pre-loading starvation if multiple streamers preload the same cache
Posted by vo...@apache.org.
ignite-3372: IgniteDataStreamer: pre-loading starvation if multiple streamers preload the same cache
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ac092f41
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ac092f41
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ac092f41
Branch: refs/heads/ignite-3410
Commit: ac092f4160f4fb7481f231f7e4de619eb4fd8f44
Parents: 76c76f8
Author: Denis Magda <dm...@gridgain.com>
Authored: Tue Jun 28 09:52:13 2016 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Tue Jun 28 09:52:13 2016 +0300
----------------------------------------------------------------------
.../cache/distributed/dht/GridDhtPartitionTopologyImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac092f41/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 2ab8a12..501388c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -652,7 +652,7 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
* ===================================================
*/
- GridDhtLocalPartition loc = localPartition(cctx.affinity().partition(e.key()), topologyVersion(), false);
+ GridDhtLocalPartition loc = localPartition(e.partition(), topologyVersion(), false);
if (loc != null)
loc.onRemoved(e);
[10/20] ignite git commit: Yardstick backport from master
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/ModelUtil.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/ModelUtil.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/ModelUtil.java
new file mode 100644
index 0000000..e21e6b4
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/ModelUtil.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.load.model;
+
+import java.util.UUID;
+import org.apache.ignite.yardstick.cache.load.model.key.Identifier;
+import org.apache.ignite.yardstick.cache.load.model.key.Mark;
+import org.apache.ignite.yardstick.cache.load.model.value.Car;
+import org.apache.ignite.yardstick.cache.load.model.value.Color;
+import org.apache.ignite.yardstick.cache.load.model.value.Truck;
+import org.apache.ignite.yardstick.cache.model.Account;
+import org.apache.ignite.yardstick.cache.model.Organization;
+import org.apache.ignite.yardstick.cache.model.Person;
+import org.apache.ignite.yardstick.cache.model.Person1;
+import org.apache.ignite.yardstick.cache.model.Person2;
+import org.apache.ignite.yardstick.cache.model.Person8;
+
+/**
+ * Util class for model.
+ */
+public class ModelUtil {
+ /**
+ * Classes of keys.
+ */
+ private static Class[] keyClasses = {
+ Double.class,
+ Identifier.class,
+ Mark.class,
+ Integer.class,
+ UUID.class
+ };
+
+ /**
+ * Classes of values.
+ */
+ private static Class[] valClasses = {
+// Car.class,
+// Truck.class,
+ Person.class,
+ Organization.class,
+ Account.class,
+ Person1.class,
+ Person2.class,
+ Person8.class
+ };
+
+ /**
+ * Simple of classes.
+ * Upper approximate size of value 24 bytes.
+ */
+ private static Class[] simpleClasses = {
+ Account.class,
+ Person1.class,
+ Person2.class,
+ };
+
+ /**
+ * Fat of classes.
+ * Upper approximate size of value 128 bytes.
+ */
+ private static Class[] fatClasses = {
+ Person.class,
+ Person8.class,
+ Organization.class
+ };
+
+ /**
+ * @param clazz Checked class.
+ * @return Result.
+ */
+ public static boolean canCreateInstance(Class clazz) {
+ for (Class c: keyClasses) {
+ if (c.equals(clazz))
+ return true;
+ }
+
+ for (Class c: valClasses) {
+ if (c.equals(clazz))
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param c model class
+ * @param id object id
+ * @return object from model
+ */
+ public static Object create(Class c, int id) {
+ Object res = null;
+
+ switch (c.getSimpleName()) {
+ case "Double":
+ res = id;
+ break;
+ case "Identifier":
+ res = new Identifier(id, "id " + id);
+ break;
+ case "Mark":
+ res = new Mark(id, UUID.nameUUIDFromBytes(Integer.toString(id).getBytes()));
+ break;
+ case "Integer":
+ res = id;
+ break;
+ case "UUID":
+ res = UUID.nameUUIDFromBytes(Integer.toString(id).getBytes());
+ break;
+ case "Car":
+ int colorCnt = Color.values().length;
+ res = new Car(id, "Mark " + id, id / 2.123 * 100, Color.values()[id % colorCnt]);
+ break;
+ case "Truck":
+ int colors = Color.values().length;
+ res = new Truck(id, "Mark " + id, id / 2.123 * 100, Color.values()[id % colors], id / 4.123 * 100);
+ break;
+ case "Person":
+ res = new Person(id, id+1, "First Name " + id, "Last Name " + id, id / 2.123 * 100);
+ break;
+ case "Organization":
+ res = new Organization(id, "Organization " + id);
+ break;
+ case "Account":
+ res = new Account(id);
+ break;
+ case "Person1":
+ res = new Person1(id);
+ break;
+ case "Person2":
+ res = new Person2(id);
+ break;
+ case "Person8":
+ res = new Person8(id);
+ }
+
+ return res;
+ }
+
+ /**
+ * @return array of key cache classes
+ */
+ public static Class[] keyClasses() {
+ return keyClasses;
+ }
+
+ /**
+ * @return Array of cache value classes
+ */
+ public static Class[] valueClasses() {
+ return valClasses;
+ }
+
+ /**
+ * @return Array of cache value simple classes.
+ */
+ public static Class[] simpleValueClasses() {
+ return simpleClasses;
+ }
+
+ /**
+ * @return Array of cache value fat classes.
+ */
+ public static Class[] fatValueClasses() {
+ return fatClasses;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/key/Identifier.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/key/Identifier.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/key/Identifier.java
new file mode 100644
index 0000000..c7c7f2d
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/key/Identifier.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.load.model.key;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Key cache class
+ */
+public class Identifier implements Comparable<Identifier>, Serializable {
+ /**
+ * Integer identifier
+ */
+ private int id;
+
+ /**
+ * String identifier
+ */
+ private String code;
+
+ /**
+ * Empty constructor
+ */
+ public Identifier() {
+ }
+
+ /**
+ * @param id identifier
+ * @param code code
+ */
+ public Identifier(int id, String code) {
+ this.id = id;
+ this.code = code;
+ }
+
+ /**
+ * @return integer identifier
+ */
+ public int getId() {
+
+ return id;
+ }
+
+ /**
+ * @param id integer identifier
+ */
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ /**
+ * @return string identifier
+ */
+ public String getCode() {
+ return code;
+ }
+
+ /**
+ * @param code string identifier
+ */
+ public void setCode(String code) {
+ this.code = code;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ Identifier that = (Identifier)o;
+
+ if (id != that.id)
+ return false;
+
+ return code != null ? code.equals(that.code) : that.code == null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = id;
+ res = 31 * res + (code != null ? code.hashCode() : 0);
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(Identifier o) {
+ return Integer.compare(id, o.id);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Identifier.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/key/Mark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/key/Mark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/key/Mark.java
new file mode 100644
index 0000000..35acef3
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/key/Mark.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.load.model.key;
+
+import java.io.Serializable;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Cache value class
+ */
+public class Mark implements Comparable<Mark>, Serializable {
+ /**
+ * Mark identifier
+ */
+ private int id;
+
+ /**
+ * Mark token
+ */
+ private UUID token;
+
+ /**
+ * Empty constructor
+ */
+ public Mark() {
+ // No-op.
+ }
+
+ /**
+ * @param id identifier
+ * @param token token
+ */
+ public Mark(int id, UUID token) {
+ this.id = id;
+ this.token = token;
+ }
+
+ /**
+ * @return mark identifier
+ */
+ public int getId() {
+ return id;
+ }
+
+ /**
+ * @param id mark identifier
+ */
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ /**
+ * @return mark token
+ */
+ public UUID getToken() {
+ return token;
+ }
+
+ /**
+ * @param token mark token
+ */
+ public void setToken(UUID token) {
+ this.token = token;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ Mark mark = (Mark)o;
+
+ if (id != mark.id)
+ return false;
+
+ return token != null ? token.equals(mark.token) : mark.token == null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = id;
+
+ res = 31 * res + (token != null ? token.hashCode() : 0);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(Mark o) {
+ return id - o.id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Mark.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Car.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Car.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Car.java
new file mode 100644
index 0000000..32be132
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Car.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.load.model.value;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Value cache class
+ */
+public class Car implements Serializable {
+ /**
+ * Car identifier
+ */
+ int id;
+
+ /**
+ * Car make
+ */
+ String make;
+
+ /**
+ * Car weight
+ */
+ double weight;
+
+ /**
+ * Car color
+ */
+ Color color;
+
+ /**
+ * Empty constructor
+ */
+ public Car() {
+ }
+
+ /**
+ * @param id identifier
+ * @param make mark of car
+ * @param weight weight
+ * @param color car color
+ */
+ public Car(int id, String make, double weight, Color color) {
+ this.id = id;
+ this.make = make;
+ this.weight = weight;
+ this.color = color;
+ }
+
+ /**
+ * @return car identifier
+ */
+ public int getId() {
+ return id;
+ }
+
+ /**
+ * @param id car identifier
+ */
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ /**
+ * @return car make
+ */
+ public String getMake() {
+ return make;
+ }
+
+ /**
+ * @param make car make
+ */
+ public void setMake(String make) {
+ this.make = make;
+ }
+
+ /**
+ * @return car weight
+ */
+ public double getWeight() {
+ return weight;
+ }
+
+ /**
+ * @param weight car weight
+ */
+ public void setWeight(double weight) {
+ this.weight = weight;
+ }
+
+ /**
+ * @return color
+ */
+ public Color getColor() {
+ return color;
+ }
+
+ /**
+ * @param color color
+ */
+ public void setColor(Color color) {
+ this.color = color;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Car.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Color.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Color.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Color.java
new file mode 100644
index 0000000..ecb3178
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Color.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.load.model.value;
+
+import java.io.Serializable;
+
+/**
+ * Car color
+ */
+public enum Color implements Serializable {
+ /**
+ * Grey color.
+ */
+ GREY,
+
+ /**
+ * Red color.
+ */
+ RED,
+
+ /**
+ * Green color.
+ */
+ GREEN,
+
+ /**
+ * Purple color.
+ */
+ PURPLE,
+
+ /**
+ * Light blue color.
+ */
+ LIGHTBLUE
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Truck.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Truck.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Truck.java
new file mode 100644
index 0000000..fd68562
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/value/Truck.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.load.model.value;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Cache value class
+ */
+public class Truck extends Car implements Serializable {
+ /**
+ * Truck capacity
+ */
+ public double cap;
+
+ /**
+ * Empty constructor
+ */
+ public Truck() {
+ // No-op.
+ }
+
+ /**
+ * @param id identifier
+ * @param make mark of truck
+ * @param weight weight
+ * @param color Color.
+ * @param cap capacity
+ */
+ public Truck(int id, String make, double weight, Color color, double cap) {
+ super(id, make, weight, color);
+ this.cap = cap;
+ }
+
+ /**
+ * @return truck capacity
+ */
+ public double getCap() {
+ return cap;
+ }
+
+ /**
+ * @param cap truck capacity
+ */
+ public void setCap(double cap) {
+ this.cap = cap;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Truck.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Account.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Account.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Account.java
index 3d5d9f8..a857b9d 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Account.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Account.java
@@ -18,6 +18,7 @@
package org.apache.ignite.yardstick.cache.model;
import java.io.Serializable;
+import org.apache.ignite.internal.util.typedef.internal.S;
/**
*
@@ -39,4 +40,9 @@ public class Account implements Serializable {
public int balance() {
return balance;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Account.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person1.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person1.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person1.java
index 9e55850..5f412f4 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person1.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person1.java
@@ -49,6 +49,6 @@ public class Person1 implements Serializable {
/** {@inheritDoc} */
@Override public String toString() {
- return "IndexedValue1 [val1=" + val1 + ']';
+ return "Person1 [val1=" + val1 + ']';
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person2.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person2.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person2.java
index cf9628c..571fe83 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person2.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/Person2.java
@@ -61,6 +61,6 @@ public class Person2 implements Serializable {
/** {@inheritDoc} */
@Override public String toString() {
- return "IndexedValue2 [val1=" + val1 + ", val2=" + val2 + ']';
+ return "Person2 [val1=" + val1 + ", val2=" + val2 + ']';
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleKey.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleKey.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleKey.java
index e089696..ac4b924 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleKey.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleKey.java
@@ -100,6 +100,6 @@ public class SampleKey implements Externalizable, Binarylizable {
/** {@inheritDoc} */
@Override public String toString() {
- return "Key [id=" + id + ']';
+ return "SampleKey [id=" + id + ']';
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleValue.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleValue.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleValue.java
index b28862c..20be370 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleValue.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/model/SampleValue.java
@@ -81,6 +81,6 @@ public class SampleValue implements Externalizable, Binarylizable {
/** {@inheritDoc} */
@Override public String toString() {
- return "Value [id=" + id + ']';
+ return "SampleValue [id=" + id + ']';
}
}
\ No newline at end of file
[11/20] ignite git commit: Yardstick backport from master
Posted by vo...@apache.org.
Yardstick backport from master
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4670bc53
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4670bc53
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4670bc53
Branch: refs/heads/ignite-3410
Commit: 4670bc53eebafa8b0f7ef053097c7c5cb5136aa5
Parents: 1873fd8
Author: Anton Vinogradov <av...@apache.org>
Authored: Wed Jun 29 16:05:09 2016 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed Jun 29 16:05:09 2016 +0300
----------------------------------------------------------------------
.../config/benchmark-cache-load-win.properties | 60 +
.../config/benchmark-cache-load.properties | 92 ++
.../yardstick/config/benchmark-full.properties | 116 ++
.../config/ignite-base-load-config.xml | 255 ++++
.../config/ignite-cache-load-config.xml | 71 ++
modules/yardstick/config/queries.sql | 2 +
modules/yardstick/pom.xml | 2 +-
.../yardstick/IgniteBenchmarkArguments.java | 50 +-
.../org/apache/ignite/yardstick/IgniteNode.java | 2 +
.../cache/IgniteCacheAbstractBenchmark.java | 59 +
.../yardstick/cache/IgniteGetTxBenchmark.java | 30 +
.../yardstick/cache/IgniteInvokeBenchmark.java | 65 +
.../cache/IgniteInvokeTxBenchmark.java | 30 +
.../IgniteCacheRandomOperationBenchmark.java | 1160 ++++++++++++++++++
.../yardstick/cache/load/model/ModelUtil.java | 181 +++
.../cache/load/model/key/Identifier.java | 113 ++
.../yardstick/cache/load/model/key/Mark.java | 115 ++
.../yardstick/cache/load/model/value/Car.java | 126 ++
.../yardstick/cache/load/model/value/Color.java | 50 +
.../yardstick/cache/load/model/value/Truck.java | 69 ++
.../ignite/yardstick/cache/model/Account.java | 6 +
.../ignite/yardstick/cache/model/Person1.java | 2 +-
.../ignite/yardstick/cache/model/Person2.java | 2 +-
.../ignite/yardstick/cache/model/SampleKey.java | 2 +-
.../yardstick/cache/model/SampleValue.java | 2 +-
25 files changed, 2656 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/config/benchmark-cache-load-win.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-cache-load-win.properties b/modules/yardstick/config/benchmark-cache-load-win.properties
new file mode 100644
index 0000000..efe24f5
--- /dev/null
+++ b/modules/yardstick/config/benchmark-cache-load-win.properties
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+::
+:: Contains benchmarks for execute random operation on various caches.
+::
+
+:: JVM options.
+set JVM_OPTS=%JVM_OPTS% -DIGNITE_QUIET=false
+
+:: Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+:: set JVM_OPTS=%JVM_OPTS%^
+:: -XX:+UseParNewGC^
+:: -XX:+UseConcMarkSweepGC^
+:: -XX:+UseTLAB^
+:: -XX:NewSize=128m^
+:: -XX:MaxNewSize=128m^
+:: -XX:MaxTenuringThreshold=0^
+:: -XX:SurvivorRatio=1024^
+:: -XX:+UseCMSInitiatingOccupancyOnly^
+:: -XX:CMSInitiatingOccupancyFraction=60
+
+:: List of default probes.
+BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe
+
+:: Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+:: Probe point writer class name.
+:: BENCHMARK_WRITER=
+
+:: Comma-separated list of the hosts to run BenchmarkServers on. 2 nodes on local host are enabled by default.
+set SERVER_HOSTS=localhost,localhost
+
+:: Comma-separated list of the hosts to run BenchmarkDrivers on. 1 node on local host is enabled by default.
+set DRIVER_HOSTS=localhost
+
+:: Remote username.
+:: set REMOTE_USER=
+
+:: Allow Operation
+set ops=PUT PUT_ALL GET GET_ALL INVOKE INVOKE_ALL REMOVE REMOVE_ALL PUT_IF_ABSENT REPLACE SCAN_QUERY SQL_QUERY CONTINUOUS_QUERY
+
+:: Run configuration which contains all benchmarks.
+:: Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+set CONFIGS=^
+-cfg %SCRIPT_DIR%\..\config\ignite-cache-load-config.xml -r 10000 -pa 500 -bs 50 -b 1 -w 20 -d 30 -t 64 -sm PRIMARY_SYNC -dn IgniteCacheRandomOperationBenchmark -sn IgniteNode -ds cache-random-benchmark-1-backup -nn 1^
+ -ltqf config\queries.sql -ltops %ops%
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/config/benchmark-cache-load.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-cache-load.properties b/modules/yardstick/config/benchmark-cache-load.properties
new file mode 100644
index 0000000..7312cb6
--- /dev/null
+++ b/modules/yardstick/config/benchmark-cache-load.properties
@@ -0,0 +1,92 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+UseTLAB \
+-XX:NewSize=128m \
+-XX:MaxNewSize=128m \
+-XX:MaxTenuringThreshold=0 \
+-XX:SurvivorRatio=1024 \
+-XX:+UseCMSInitiatingOccupancyOnly \
+-XX:CMSInitiatingOccupancyFraction=60 \
+"
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# Comma-separated list of the hosts to run BenchmarkServers on. 2 nodes on local host are enabled by default.
+SERVER_HOSTS=localhost,localhost,localhost
+
+# Comma-separated list of the hosts to run BenchmarkDrivers on. 1 node on local host is enabled by default.
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+# Preload amount
+pa=500000
+
+# Key range
+r=1000000
+
+# Batch size
+
+bs=50
+
+# Allow Operation
+ops="PUT PUT_ALL GET GET_ALL INVOKE INVOKE_ALL REMOVE REMOVE_ALL PUT_IF_ABSENT REPLACE SCAN_QUERY SQL_QUERY CONTINUOUS_QUERY"
+
+# Run configuration.
+# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-cache-load-config.xml -r ${r} -pa ${pa} -bs ${bs} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteCacheRandomOperationBenchmark -sn IgniteNode -ds cache-random-benchmark-${b}-backup -nn ${nodesNum} \
+-ltqf ${SCRIPT_DIR}/../config/queries.sql -ltops ${ops}"
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/config/benchmark-full.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-full.properties b/modules/yardstick/config/benchmark-full.properties
new file mode 100644
index 0000000..da3ae9f
--- /dev/null
+++ b/modules/yardstick/config/benchmark-full.properties
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Contains all benchmarks
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+ -Xloggc:./gc${now0}.log \
+ -XX:+PrintGCDetails \
+ -verbose:gc \
+ -XX:+UseParNewGC \
+ -XX:+UseConcMarkSweepGC \
+ -XX:+UseTLAB \
+ -XX:NewSize=128m \
+ -XX:MaxNewSize=128m \
+ -XX:MaxTenuringThreshold=0 \
+ -XX:SurvivorRatio=1024 \
+ -XX:+UseCMSInitiatingOccupancyOnly \
+ -XX:CMSInitiatingOccupancyFraction=60 \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# Comma-separated list of the hosts to run BenchmarkServers on. 2 nodes on local host are enabled by default.
+SERVER_HOSTS=localhost,localhost,localhost
+
+# Comma-separated list of the hosts to run BenchmarkDrivers on. 1 node on local host is enabled by default.
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutGetBatchBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-bs-2-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutGetBatchBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-bs-6-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutGetBatchBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-bs-10-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxImplicitBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-put-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-optim-repRead-put-get-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc PESSIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-pessim-repRead-put-get-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -txi SERIALIZABLE -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serial-put-get-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-bs-2-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-bs-6-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-bs-10-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-bs2-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-bs-6-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-bs-10-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-bs-2-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-bs-6-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-bs-10-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-bs-2-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-bs-6-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-bs-10-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-bs-2-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-bs-6-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-bs-10-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-bs-2-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-bs-6-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-bs-10-1-backup,\
+"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/config/ignite-base-load-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-base-load-config.xml b/modules/yardstick/config/ignite-base-load-config.xml
new file mode 100644
index 0000000..614f778
--- /dev/null
+++ b/modules/yardstick/config/ignite-base-load-config.xml
@@ -0,0 +1,255 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+ <bean id="base-ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" abstract="true">
+ <property name="peerClassLoadingEnabled" value="false"/>
+
+ <property name="metricsLogFrequency" value="5000"/>
+
+ <property name="includeEventTypes">
+ <list/>
+ </property>
+
+ <property name="loadBalancingSpi">
+ <bean class="org.apache.ignite.spi.loadbalancing.roundrobin.RoundRobinLoadBalancingSpi">
+ <property name="perTask" value="false"/>
+ </bean>
+ </property>
+ </bean>
+
+ <bean name="atomic" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="atomic-fat-values"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="ATOMIC"/>
+
+ <property name="swapEnabled" value="false"/>
+ </bean>
+
+ <bean name="atomic-offheap" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="atomic-offheap"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="ATOMIC"/>
+
+ <property name="swapEnabled" value="false"/>
+
+ <property name="memoryMode" value="OFFHEAP_TIERED"/>
+ </bean>
+
+ <bean name="atomic-offheap-values" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="atomic-offheap-values"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="ATOMIC"/>
+
+ <property name="swapEnabled" value="false"/>
+
+ <property name="memoryMode" value="OFFHEAP_VALUES"/>
+
+ </bean>
+
+ <bean name="tx" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="tx-fat-values"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+
+ <property name="swapEnabled" value="false"/>
+
+ </bean>
+
+ <bean name="tx-offheap" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="tx-offheap"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+
+ <property name="swapEnabled" value="false"/>
+
+ <property name="memoryMode" value="OFFHEAP_TIERED"/>
+
+ </bean>
+
+ <bean name="tx-offheap-values" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="tx-offheap-values"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+
+ <property name="swapEnabled" value="false"/>
+
+ <property name="memoryMode" value="OFFHEAP_VALUES"/>
+
+ </bean>
+
+ <bean name="atomic-index" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="atomic-index"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="ATOMIC"/>
+
+ <property name="swapEnabled" value="false"/>
+
+ <property name="indexedTypes">
+ <list>
+ <value>java.lang.Integer</value>
+ <value>org.apache.ignite.yardstick.cache.model.Person1</value>
+
+ <value>java.lang.Integer</value>
+ <value>org.apache.ignite.yardstick.cache.model.Person2</value>
+
+ <value>java.lang.Integer</value>
+ <value>org.apache.ignite.yardstick.cache.model.Person8</value>
+ </list>
+ </property>
+ </bean>
+
+ <bean name="query" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="query"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="ATOMIC"/>
+
+ <property name="swapEnabled" value="false"/>
+
+ <!-- Configure query entities -->
+ <property name="queryEntities">
+ <list>
+ <bean class="org.apache.ignite.cache.QueryEntity">
+ <property name="keyType" value="java.lang.Integer"/>
+ <property name="valueType" value="org.apache.ignite.yardstick.cache.model.Person"/>
+
+ <property name="fields">
+ <map>
+ <entry key="id" value="java.lang.Integer"/>
+ <entry key="orgId" value="java.lang.Integer"/>
+ <entry key="firstName" value="java.lang.String"/>
+ <entry key="lastName" value="java.lang.String"/>
+ <entry key="salary" value="java.lang.Double"/>
+ </map>
+ </property>
+
+ <property name="indexes">
+ <list>
+ <bean class="org.apache.ignite.cache.QueryIndex">
+ <constructor-arg value="id"/>
+ </bean>
+ <!-- Group index. -->
+ <bean class="org.apache.ignite.cache.QueryIndex">
+ <constructor-arg>
+ <list>
+ <value>id</value>
+ <value>salary</value>
+ </list>
+ </constructor-arg>
+ <constructor-arg value="SORTED"/>
+ </bean>
+ </list>
+ </property>
+ </bean>
+ </list>
+ </property>
+ </bean>
+
+ <bean name="orgCache" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="orgCache"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="ATOMIC"/>
+
+ <property name="swapEnabled" value="false"/>
+
+ <!-- Configure query entities -->
+ <property name="queryEntities">
+ <list>
+ <bean class="org.apache.ignite.cache.QueryEntity">
+ <property name="keyType" value="java.lang.Integer"/>
+ <property name="valueType" value="org.apache.ignite.yardstick.cache.model.Organization"/>
+
+ <property name="fields">
+ <map>
+ <entry key="id" value="java.lang.Integer"/>
+ <entry key="name" value="java.lang.String"/>
+ </map>
+ </property>
+
+ <property name="indexes">
+ <list>
+ <bean class="org.apache.ignite.cache.QueryIndex">
+ <constructor-arg value="id"/>
+ </bean>
+ <bean class="org.apache.ignite.cache.QueryIndex">
+ <constructor-arg value="name"/>
+ </bean>
+ </list>
+ </property>
+ </bean>
+ </list>
+ </property>
+ </bean>
+
+ <bean name="query-offheap" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="query-offheap"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="ATOMIC"/>
+
+ <property name="swapEnabled" value="false"/>
+
+ <property name="memoryMode" value="OFFHEAP_TIERED"/>
+
+ <property name="indexedTypes">
+ <list>
+ <value>java.lang.Integer</value>
+ <value>org.apache.ignite.yardstick.cache.model.Organization</value>
+
+ <value>java.lang.Integer</value>
+ <value>org.apache.ignite.yardstick.cache.model.Person</value>
+ </list>
+ </property>
+ </bean>
+
+ <bean name="compute" class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="compute"/>
+
+ <property name="cacheMode" value="PARTITIONED"/>
+
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+
+ <property name="swapEnabled" value="false"/>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/config/ignite-cache-load-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-cache-load-config.xml b/modules/yardstick/config/ignite-cache-load-config.xml
new file mode 100644
index 0000000..6c6ac89
--- /dev/null
+++ b/modules/yardstick/config/ignite-cache-load-config.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!--
+ Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+ <import resource="ignite-base-load-config.xml"/>
+
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" parent="base-ignite.cfg">
+ <property name="localHost" value="127.0.0.1"/>
+
+ <property name="cacheConfiguration">
+ <list>
+ <ref bean="atomic" />
+ <ref bean="atomic-offheap" />
+ <ref bean="atomic-offheap-values" />
+ <ref bean="tx" />
+ <ref bean="tx-offheap" />
+ <ref bean="tx-offheap-values" />
+ <ref bean="atomic-index" />
+ <ref bean="query" />
+ <ref bean="orgCache" />
+ <!--<ref bean="query-offheap" />-->
+ <ref bean="compute" />
+ </list>
+ </property>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47500</value>
+ <value>127.0.0.1:47501</value>
+ <value>127.0.0.1:47502</value>
+ <value>127.0.0.1:47503</value>
+ <value>127.0.0.1:47504</value>
+ <value>127.0.0.1:47505</value>
+ <value>127.0.0.1:47506</value>
+ <value>127.0.0.1:47507</value>
+ <value>127.0.0.1:47508</value>
+ <value>127.0.0.1:47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/config/queries.sql
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/queries.sql b/modules/yardstick/config/queries.sql
new file mode 100644
index 0000000..8271100
--- /dev/null
+++ b/modules/yardstick/config/queries.sql
@@ -0,0 +1,2 @@
+SELECT Person.firstName FROM "query".Person, "orgCache".Organization WHERE Person.orgId = Organization.id AND lower(Organization.name) = lower('Organization 55')
+SELECT Organization.name FROM "orgCache".Organization WHERE lower(Organization.name) LIKE lower('%55%')
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/pom.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index 52418d9..a0f7de5 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -31,7 +31,7 @@
</parent>
<artifactId>ignite-yardstick</artifactId>
- <version>1.6.0-SNAPSHOT</version>
+ <version>1.7.0-SNAPSHOT</version>
<url>http://ignite.apache.org</url>
<properties>
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 1ecfa0f..1854938 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -24,6 +24,9 @@ import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* Input arguments for Ignite benchmarks.
*/
@@ -37,10 +40,15 @@ public class IgniteBenchmarkArguments {
@Parameter(names = {"-b", "--backups"}, description = "Backups")
private int backups;
+ /** */
@Parameter(names = {"-cfg", "--Config"}, description = "Configuration file")
private String cfg = "config/ignite-localhost-config.xml";
/** */
+ @Parameter(names = {"-ltqf", "--loadTestQueriesFile"}, description = "File with predefined SQL queries")
+ private String loadTestQueriesFile = null;
+
+ /** */
@Parameter(names = {"-sm", "--syncMode"}, description = "Synchronization mode")
private CacheWriteSynchronizationMode syncMode = CacheWriteSynchronizationMode.PRIMARY_SYNC;
@@ -86,7 +94,11 @@ public class IgniteBenchmarkArguments {
/** */
@Parameter(names = {"-r", "--range"}, description = "Key range")
- private int range = 1_000_000;
+ public int range = 1_000_000;
+
+ /** */
+ @Parameter(names = {"-pa", "--preloadAmount"}, description = "Data pre-loading amount for load tests")
+ public int preloadAmount = 500_000;
/** */
@Parameter(names = {"-j", "--jobs"}, description = "Number of jobs for compute benchmarks")
@@ -136,6 +148,28 @@ public class IgniteBenchmarkArguments {
@Parameter(names = {"-kpt", "--keysPerThread"}, description = "Use not intersecting keys in putAll benchmark")
private boolean keysPerThread;
+ /** */
+ @Parameter(names = {"-pp", "--printPartitionStats"}, description = "Print partition statistics")
+ private boolean printPartStats;
+
+ /** */
+ @Parameter(names = {"-ltops", "--allowedLoadTestOperations"}, variableArity = true, description = "List of enabled load test operations")
+ private List<String> allowedLoadTestOps = new ArrayList<>();
+
+ /**
+ * @return List of enabled load test operations.
+ */
+ public List<String> allowedLoadTestOps() {
+ return allowedLoadTestOps;
+ }
+
+ /**
+ * @return If {@code true} when need to print partition statistics.
+ */
+ public boolean printPartitionStatistics() {
+ return printPartStats;
+ }
+
/**
* @return JDBC url.
*/
@@ -249,6 +283,13 @@ public class IgniteBenchmarkArguments {
}
/**
+ * @return Preload key range, from {@code 0} to this number.
+ */
+ public int preloadAmount() {
+ return preloadAmount;
+ }
+
+ /**
* @return Configuration file.
*/
public String configuration() {
@@ -256,6 +297,13 @@ public class IgniteBenchmarkArguments {
}
/**
+ * @return File with predefined SQL queries.
+ */
+ public String loadTestQueriesFile() {
+ return loadTestQueriesFile;
+ }
+
+ /**
* @return Number of jobs
*/
public int jobs() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteNode.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteNode.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteNode.java
index 33656fb..eed4450 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteNode.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteNode.java
@@ -136,6 +136,8 @@ public class IgniteNode implements BenchmarkServer {
cc.setWriteThrough(args.isStoreEnabled());
cc.setWriteBehindEnabled(args.isWriteBehind());
+
+ BenchmarkUtils.println(cfg, "Cache configured with the following parameters: " + cc);
}
TransactionConfiguration tc = c.getTransactionConfiguration();
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
index 3efa4a5..da5cb1d 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
@@ -17,9 +17,19 @@
package org.apache.ignite.yardstick.cache;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
import org.yardstickframework.BenchmarkConfiguration;
import org.yardstickframework.BenchmarkUtils;
@@ -42,6 +52,55 @@ public abstract class IgniteCacheAbstractBenchmark<K, V> extends IgniteAbstractB
super.setUp(cfg);
cache = cache();
+
+ BenchmarkUtils.println(cfg, "Benchmark setUp [name=" + getClass().getSimpleName() +
+ ", cacheName="+ cache.getName() +
+ ", cacheCfg=" + cache.getConfiguration(CacheConfiguration.class) + ']');
+
+ if (args.printPartitionStatistics()) {
+ Map<ClusterNode, T2<List<Integer>, List<Integer>>> parts = new HashMap<>();
+
+ for (ClusterNode node : ignite().cluster().nodes())
+ parts.put(node,
+ new T2<List<Integer>, List<Integer>>(new ArrayList<Integer>(), new ArrayList<Integer>()));
+
+ U.sleep(5000);
+
+ Affinity<Object> aff = ignite().affinity(cache.getName());
+
+ for (int p = 0; p < aff.partitions(); p++) {
+ Collection<ClusterNode> nodes = aff.mapPartitionToPrimaryAndBackups(p);
+
+ boolean primary = true;
+
+ for (ClusterNode node : nodes) {
+ if (primary) {
+ parts.get(node).get1().add(p);
+
+ primary = false;
+ }
+ else
+ parts.get(node).get2().add(p);
+ }
+ }
+
+ BenchmarkUtils.println(cfg, "Partition stats. [cacheName: "+ cache.getName() +", topVer: "
+ + ignite().cluster().topologyVersion() + "]");
+ BenchmarkUtils.println(cfg, "(Node id, Number of Primary, Percent, Number of Backup, Percent, Total, Percent)");
+
+ for (Map.Entry<ClusterNode, T2<List<Integer>, List<Integer>>> e : parts.entrySet()) {
+ List<Integer> primary = e.getValue().get1();
+ List<Integer> backup = e.getValue().get2();
+
+ BenchmarkUtils.println(cfg, e.getKey().id() + " "
+ + primary.size() + " " + primary.size() * 1. /aff.partitions() + " "
+ + backup.size() + " "
+ + backup.size() * 1. / (aff.partitions() * (args.backups() == 0 ? 1 : args.backups())) + " "
+ + (primary.size() + backup.size()) + " "
+ + (primary.size() + backup.size() * 1.) / (aff.partitions() * args.backups() + aff.partitions())
+ );
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetTxBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetTxBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetTxBenchmark.java
new file mode 100644
index 0000000..fbb73e2
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetTxBenchmark.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import org.apache.ignite.IgniteCache;
+
+/**
+ *
+ */
+public class IgniteGetTxBenchmark extends IgniteGetBenchmark {
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Integer, Object> cache() {
+ return ignite().cache("tx");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteInvokeBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteInvokeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteInvokeBenchmark.java
new file mode 100644
index 0000000..a1e80f0
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteInvokeBenchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Map;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.yardstick.cache.model.SampleValue;
+
+/**
+ * Ignite benchmark that performs invoke operations.
+ */
+public class IgniteInvokeBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ int key = nextRandom(args.range());
+
+ cache.invoke(key, new SetValueEntryProcessor(new SampleValue(key)));
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Integer, Object> cache() {
+ return ignite().cache("atomic");
+ }
+
+ /**
+ *
+ */
+ public static class SetValueEntryProcessor implements CacheEntryProcessor<Integer, Object, Object> {
+ /** */
+ private Object val;
+
+ /**
+ * @param val Value.
+ */
+ public SetValueEntryProcessor(Object val) {
+ this.val = val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object process(MutableEntry<Integer, Object> entry, Object... args) {
+ entry.setValue(val);
+
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteInvokeTxBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteInvokeTxBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteInvokeTxBenchmark.java
new file mode 100644
index 0000000..8f05598
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteInvokeTxBenchmark.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import org.apache.ignite.IgniteCache;
+
+/**
+ * Ignite benchmark that performs invoke operations.
+ */
+public class IgniteInvokeTxBenchmark extends IgniteInvokeBenchmark {
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Integer, Object> cache() {
+ return ignite().cache("tx");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4670bc53/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
new file mode 100644
index 0000000..e676b38
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
@@ -0,0 +1,1160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.load;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.cache.configuration.FactoryBuilder;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheEntryEventSerializableFilter;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheTypeMetadata;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.Query;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+import org.apache.ignite.yardstick.IgniteBenchmarkUtils;
+import org.apache.ignite.yardstick.cache.load.model.ModelUtil;
+import org.jetbrains.annotations.NotNull;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Ignite cache random operation benchmark.
+ */
+public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark {
+ /** */
+ public static final int operations = Operation.values().length;
+
+ /***/
+ public static final int CONTINUOUS_QUERY_PER_CACHE = 3;
+
+ /** Scan query predicate. */
+ private static BenchmarkIgniteBiPredicate igniteBiPred = new BenchmarkIgniteBiPredicate();
+
+ /** Amount partitions. */
+ private static final int SCAN_QUERY_PARTITION_AMOUNT = 10;
+
+ /** List off all available cache. */
+ private List<IgniteCache<Object, Object>> availableCaches;
+
+ /** List of available transactional cache. */
+ private List<IgniteCache<Object, Object>> txCaches;
+
+ /** List of affinity cache. */
+ private List<IgniteCache<Object, Object>> affCaches;
+
+ /** Map cache name on key classes. */
+ private Map<String, Class[]> keysCacheClasses;
+
+ /** Map cache name on value classes. */
+ private Map<String, Class[]> valuesCacheClasses;
+
+ /** List of query descriptors by cache names. */
+ private Map<String, List<SqlCacheDescriptor>> cacheSqlDescriptors;
+
+ /** List of SQL queries. */
+ private List<String> queries;
+
+ /** List of allowed cache operations which will be executed. */
+ private List<Operation> allowedLoadTestOps;
+
+ /**
+ * Replace value entry processor.
+ */
+ private BenchmarkReplaceValueEntryProcessor replaceEntryProc;
+
+ /**
+ * Remove entry processor.
+ */
+ private BenchmarkRemoveEntryProcessor rmvEntryProc;
+
+ /**
+ * Map of statistic information.
+ */
+ private Map<String, AtomicLong> operationStatistics;
+
+ /** {@inheritDoc} */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ searchCache();
+
+ preLoading();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onException(Throwable e) {
+ BenchmarkUtils.errorHelp(cfg, "The benchmark of random operation failed.");
+ super.onException(e);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> map) throws Exception {
+ if (nextBoolean()) {
+ executeInTransaction(map);
+
+ executeOutOfTx(map, true);
+ }
+ else
+ executeOutOfTx(map, false);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void tearDown() throws Exception {
+ try {
+ BenchmarkUtils.println("Benchmark statistics");
+
+ for (String cacheName : ignite().cacheNames()) {
+ BenchmarkUtils.println(String.format("Operations over cache '%s'", cacheName));
+
+ for (Operation op : Operation.values()) {
+ BenchmarkUtils.println(cfg, String.format("%s: %s", op,
+ operationStatistics.get(op + "_" + cacheName)));
+ }
+ }
+ }
+ finally {
+ super.tearDown();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void searchCache() throws Exception {
+ availableCaches = new ArrayList<>(ignite().cacheNames().size());
+ txCaches = new ArrayList<>();
+ affCaches = new ArrayList<>();
+ keysCacheClasses = new HashMap<>();
+ valuesCacheClasses = new HashMap<>();
+ replaceEntryProc = new BenchmarkReplaceValueEntryProcessor(null);
+ rmvEntryProc = new BenchmarkRemoveEntryProcessor();
+ cacheSqlDescriptors = new HashMap<>();
+ operationStatistics = new HashMap<>();
+
+ loadQueries();
+
+ loadAllowedOperations();
+
+ for (String cacheName : ignite().cacheNames()) {
+ IgniteCache<Object, Object> cache = ignite().cache(cacheName);
+
+ for (Operation op : Operation.values())
+ operationStatistics.put(op + "_" + cacheName, new AtomicLong(0));
+
+ CacheConfiguration configuration = cache.getConfiguration(CacheConfiguration.class);
+
+ if (isClassDefinedInConfig(configuration)) {
+ if (configuration.getMemoryMode() == CacheMemoryMode.OFFHEAP_TIERED &&
+ configuration.getQueryEntities().size() > 2) {
+ throw new IgniteException("Off-heap mode is unsupported by the load test due to bugs IGNITE-2982" +
+ " and IGNITE-2997");
+ }
+
+ ArrayList<Class> keys = new ArrayList<>();
+ ArrayList<Class> values = new ArrayList<>();
+
+ if (configuration.getQueryEntities() != null) {
+ Collection<QueryEntity> entries = configuration.getQueryEntities();
+
+ for (QueryEntity queryEntity : entries) {
+ if (queryEntity.getKeyType() != null) {
+ Class keyCls = Class.forName(queryEntity.getKeyType());
+
+ if (ModelUtil.canCreateInstance(keyCls))
+ keys.add(keyCls);
+ else
+ throw new IgniteException("Class is unknown for the load test. Make sure you " +
+ "specified its full name [clsName=" + queryEntity.getKeyType() + ']');
+ }
+
+ if (queryEntity.getValueType() != null) {
+ Class valCls = Class.forName(queryEntity.getValueType());
+
+ if (ModelUtil.canCreateInstance(valCls))
+ values.add(valCls);
+ else
+ throw new IgniteException("Class is unknown for the load test. Make sure you " +
+ "specified its full name [clsName=" + queryEntity.getKeyType() + ']');
+
+ cofigureCacheSqlDescriptor(cacheName, queryEntity, valCls);
+ }
+ }
+ }
+
+ if (configuration.getTypeMetadata() != null) {
+ Collection<CacheTypeMetadata> entries = configuration.getTypeMetadata();
+
+ for (CacheTypeMetadata cacheTypeMetadata : entries) {
+ if (cacheTypeMetadata.getKeyType() != null) {
+ Class keyCls = Class.forName(cacheTypeMetadata.getKeyType());
+
+ if (ModelUtil.canCreateInstance(keyCls))
+ keys.add(keyCls);
+ else
+ throw new IgniteException("Class is unknown for the load test. Make sure you " +
+ "specified its full name [clsName=" + cacheTypeMetadata.getKeyType() + ']');
+ }
+
+ if (cacheTypeMetadata.getValueType() != null) {
+ Class valCls = Class.forName(cacheTypeMetadata.getValueType());
+
+ if (ModelUtil.canCreateInstance(valCls))
+ values.add(valCls);
+ else
+ throw new IgniteException("Class is unknown for the load test. Make sure you " +
+ "specified its full name [clsName=" + cacheTypeMetadata.getKeyType() + ']');
+ }
+ }
+ }
+
+ if (keys.isEmpty() || values.isEmpty())
+ continue;
+
+ keysCacheClasses.put(cacheName, keys.toArray(new Class[] {}));
+
+ valuesCacheClasses.put(cacheName, values.toArray(new Class[] {}));
+ }
+ else
+ keysCacheClasses.put(cacheName, new Class[] {randomKeyClass(cacheName)});
+
+ valuesCacheClasses.put(cacheName, determineValueClasses(cacheName));
+
+ if (configuration.getCacheMode() != CacheMode.LOCAL)
+ affCaches.add(cache);
+
+ if (configuration.getAtomicityMode() == CacheAtomicityMode.TRANSACTIONAL)
+ txCaches.add(cache);
+
+ availableCaches.add(cache);
+ }
+ }
+
+ /**
+ * Load allowed operation from parameters.
+ */
+ private void loadAllowedOperations() {
+ allowedLoadTestOps = new ArrayList<>();
+ if (args.allowedLoadTestOps().isEmpty())
+ Collections.addAll(allowedLoadTestOps, Operation.values());
+ else
+ for (String opName : args.allowedLoadTestOps())
+ allowedLoadTestOps.add(Operation.valueOf(opName.toUpperCase()));
+ }
+
+ /**
+ * Load query from file.
+ *
+ * @throws Exception If fail.
+ */
+ private void loadQueries() throws Exception {
+ queries = new ArrayList<>();
+
+ if (args.loadTestQueriesFile() != null) {
+ try (FileReader fr = new FileReader(args.loadTestQueriesFile())) {
+ try (BufferedReader br = new BufferedReader(fr)) {
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.trim().isEmpty())
+ continue;
+
+ queries.add(line.trim());
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * @param cacheName Ignite cache name.
+ * @param qryEntity Query entry.
+ * @param valCls Class of value.
+ * @throws ClassNotFoundException If fail.
+ */
+ private void cofigureCacheSqlDescriptor(String cacheName, QueryEntity qryEntity, Class valCls)
+ throws ClassNotFoundException {
+ List<SqlCacheDescriptor> descs = cacheSqlDescriptors.get(cacheName);
+
+ if (descs == null) {
+ descs = new ArrayList<>();
+
+ cacheSqlDescriptors.put(cacheName, descs);
+ }
+
+ Map<String, Class> indexedFields = new HashMap<>();
+
+ for (QueryIndex index : qryEntity.getIndexes()) {
+ for (String iField : index.getFieldNames()) {
+ indexedFields.put(iField,
+ Class.forName(qryEntity.getFields().get(iField)));
+ }
+ }
+
+ descs.add(new SqlCacheDescriptor(valCls, qryEntity.getFields().keySet(),
+ indexedFields));
+ }
+
+ /**
+ * @param configuration Ignite cache configuration.
+ * @return True if defined.
+ */
+ private boolean isClassDefinedInConfig(CacheConfiguration configuration) {
+ return (configuration.getIndexedTypes() != null && configuration.getIndexedTypes().length > 0)
+ || !CollectionUtils.isEmpty(configuration.getQueryEntities())
+ || !CollectionUtils.isEmpty(configuration.getTypeMetadata());
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ private void preLoading() throws Exception {
+ if (args.preloadAmount() > args.range())
+ throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\") must by less then the" +
+ " range (\"-r\", \"--range\").");
+
+ Thread[] threads = new Thread[availableCaches.size()];
+
+ for (int i = 0; i < availableCaches.size(); i++) {
+ final String cacheName = availableCaches.get(i).getName();
+
+ threads[i] = new Thread() {
+ @Override public void run() {
+ try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(cacheName)) {
+ for (int i = 0; i < args.preloadAmount() && !isInterrupted(); i++)
+ dataLdr.addData(createRandomKey(i, cacheName), createRandomValue(i, cacheName));
+ }
+ }
+ };
+
+ threads[i].start();
+ }
+
+ for (Thread thread : threads)
+ thread.join();
+ }
+
+ /**
+ * Building a map that contains mapping of node ID to a list of partitions stored on the node.
+ *
+ * @param cacheName Name of Ignite cache.
+ * @return Node to partitions map.
+ */
+ private Map<UUID, List<Integer>> personCachePartitions(String cacheName) {
+ // Getting affinity for person cache.
+ Affinity<Object> affinity = ignite().affinity(cacheName);
+
+ // Building a list of all partitions numbers.
+ List<Integer> rndParts = new ArrayList<>(10);
+
+ if (affinity.partitions() <= SCAN_QUERY_PARTITION_AMOUNT)
+ for (int i = 0; i < affinity.partitions(); i++)
+ rndParts.add(i);
+ else {
+ for (int i = 0; i < SCAN_QUERY_PARTITION_AMOUNT; i++) {
+ int partNum;
+
+ do
+ partNum = nextRandom(affinity.partitions());
+ while (rndParts.contains(partNum));
+
+ rndParts.add(partNum);
+ }
+ }
+
+ Collections.sort(rndParts);
+
+ // Getting partition to node mapping.
+ Map<Integer, ClusterNode> partPerNodes = affinity.mapPartitionsToNodes(rndParts);
+
+ // Building node to partitions mapping.
+ Map<UUID, List<Integer>> nodesToPart = new HashMap<>();
+
+ for (Map.Entry<Integer, ClusterNode> entry : partPerNodes.entrySet()) {
+ List<Integer> nodeParts = nodesToPart.get(entry.getValue().id());
+
+ if (nodeParts == null) {
+ nodeParts = new ArrayList<>();
+
+ nodesToPart.put(entry.getValue().id(), nodeParts);
+ }
+
+ nodeParts.add(entry.getKey());
+ }
+
+ return nodesToPart;
+ }
+
+ /**
+ * @param id Object identifier.
+ * @param cacheName Name of Ignite cache.
+ * @return Random object.
+ */
+ private Object createRandomKey(int id, String cacheName) {
+ Class clazz = randomKeyClass(cacheName);
+
+ return ModelUtil.create(clazz, id);
+ }
+
+ /**
+ * @param cacheName Ignite cache name.
+ * @return Random key class.
+ */
+ private Class randomKeyClass(String cacheName) {
+ Class[] keys = keysCacheClasses.containsKey(cacheName)
+ ? keysCacheClasses.get(cacheName) : ModelUtil.keyClasses();
+
+ return keys[nextRandom(keys.length)];
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @return Set classes for cache.
+ */
+ private Class[] determineValueClasses(@NotNull String cacheName) {
+ return cacheName.toLowerCase().contains("fat-values") ? ModelUtil.fatValueClasses() :
+ ModelUtil.simpleValueClasses();
+ }
+
+ /**
+ * @param id Object identifier.
+ * @param cacheName Name of Ignite cache.
+ * @return Random object.
+ */
+ private Object createRandomValue(int id, String cacheName) {
+ Class clazz = randomValueClass(cacheName);
+
+ return ModelUtil.create(clazz, id);
+ }
+
+ /**
+ * @param cacheName Ignite cache name.
+ * @return Random value class.
+ */
+ private Class randomValueClass(String cacheName) {
+ Class[] values = valuesCacheClasses.containsKey(cacheName)
+ ? valuesCacheClasses.get(cacheName) : ModelUtil.valueClasses();
+
+ return values[nextRandom(values.length)];
+ }
+
+ /**
+ * @param map Parameters map.
+ * @param withoutTransactionCache Without transaction cache.
+ * @throws Exception If fail.
+ */
+ private void executeOutOfTx(Map<Object, Object> map, boolean withoutTransactionCache) throws Exception {
+ for (IgniteCache<Object, Object> cache : availableCaches) {
+ if (withoutTransactionCache && txCaches.contains(cache))
+ continue;
+
+ executeRandomOperation(map, cache);
+ }
+ }
+
+ /**
+ * @param map Parameters map.
+ * @param cache Ignite cache.
+ * @throws Exception If fail.
+ */
+ private void executeRandomOperation(Map<Object, Object> map, IgniteCache<Object, Object> cache) throws Exception {
+ Operation op = nextRandomOperation();
+
+ switch (op) {
+ case PUT:
+ doPut(cache);
+ break;
+
+ case PUT_ALL:
+ doPutAll(cache);
+ break;
+
+ case GET:
+ doGet(cache);
+ break;
+
+ case GET_ALL:
+ doGetAll(cache);
+ break;
+
+ case INVOKE:
+ doInvoke(cache);
+ break;
+
+ case INVOKE_ALL:
+ doInvokeAll(cache);
+ break;
+
+ case REMOVE:
+ doRemove(cache);
+ break;
+
+ case REMOVE_ALL:
+ doRemoveAll(cache);
+ break;
+
+ case PUT_IF_ABSENT:
+ doPutIfAbsent(cache);
+ break;
+
+ case REPLACE:
+ doReplace(cache);
+ break;
+
+ case SCAN_QUERY:
+ doScanQuery(cache);
+ break;
+
+ case SQL_QUERY:
+ doSqlQuery(cache);
+ break;
+
+ case CONTINUOUS_QUERY:
+ doContinuousQuery(cache, map);
+ }
+ storeStatistics(cache.getName(), map, op);
+ }
+
+ /**
+ * @return Operation.
+ */
+ @NotNull private Operation nextRandomOperation() {
+ return allowedLoadTestOps.get(nextRandom(allowedLoadTestOps.size()));
+ }
+
+ /**
+ * @param cacheName Ignite cache name.
+ * @param map Parameters map.
+ * @param op Operation.
+ */
+ private void storeStatistics(String cacheName, Map<Object, Object> map, Operation op) {
+ String opCacheKey = op + "_" + cacheName;
+
+ Long opAmount = (Long)map.get("amount");
+
+ opAmount = opAmount == null ? 1L : opAmount + 1;
+
+ Integer opCacheCnt = (Integer)map.get(opCacheKey);
+
+ opCacheCnt = opCacheCnt == null ? 1 : opCacheCnt + 1;
+
+ if (opAmount % 100 == 0)
+ updateStat(map);
+ else
+ map.put(opCacheKey, opCacheCnt);
+
+ map.put("amount", opAmount);
+ }
+
+ /**
+ * @param map Parameters map.
+ */
+ private void updateStat(Map<Object, Object> map) {
+ for (Operation op: Operation.values())
+ for (String cacheName: ignite().cacheNames()) {
+ String opCacheKey = op + "_" + cacheName;
+
+ Integer val = (Integer)map.get(opCacheKey);
+
+ if (val != null) {
+ operationStatistics.get(opCacheKey).addAndGet(val.longValue());
+
+ map.put(opCacheKey, 0);
+ }
+
+ }
+ }
+
+ /**
+ * Execute operations in transaction.
+ * @param map Parameters map.
+ * @throws Exception if fail.
+ */
+ private void executeInTransaction(final Map<Object, Object> map) throws Exception {
+ IgniteBenchmarkUtils.doInTransaction(ignite().transactions(),
+ TransactionConcurrency.fromOrdinal(nextRandom(TransactionConcurrency.values().length)),
+ TransactionIsolation.fromOrdinal(nextRandom(TransactionIsolation.values().length)),
+
+ new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ for (IgniteCache<Object, Object> cache : txCaches)
+ if (nextBoolean())
+ executeRandomOperation(map, cache);
+
+ return null;
+ }
+ });
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doPut(IgniteCache<Object, Object> cache) throws Exception {
+ int i = nextRandom(args.range());
+
+ cache.put(createRandomKey(i, cache.getName()), createRandomValue(i, cache.getName()));
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doPutAll(IgniteCache<Object, Object> cache) throws Exception {
+ Map<Object, Object> putMap = new TreeMap<>();
+
+ Class keyCass = randomKeyClass(cache.getName());
+
+ for (int cnt = 0; cnt < args.batch(); cnt++) {
+ int i = nextRandom(args.range());
+
+ putMap.put(ModelUtil.create(keyCass, i), createRandomValue(i, cache.getName()));
+ }
+
+ cache.putAll(putMap);
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doGet(IgniteCache<Object, Object> cache) throws Exception {
+ int i = nextRandom(args.range());
+
+ cache.get(createRandomKey(i, cache.getName()));
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doGetAll(IgniteCache<Object, Object> cache) throws Exception {
+ Set<Object> keys = new TreeSet<>();
+
+ Class keyCls = randomKeyClass(cache.getName());
+
+ for (int cnt = 0; cnt < args.batch(); cnt++) {
+ int i = nextRandom(args.range());
+
+ keys.add(ModelUtil.create(keyCls, i));
+ }
+
+ cache.getAll(keys);
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doInvoke(final IgniteCache<Object, Object> cache) throws Exception {
+ final int i = nextRandom(args.range());
+
+ if (nextBoolean())
+ cache.invoke(createRandomKey(i, cache.getName()), replaceEntryProc,
+ createRandomValue(i + 1, cache.getName()));
+ else
+ cache.invoke(createRandomKey(i, cache.getName()), rmvEntryProc);
+
+ }
+
+ /**
+ * Entry processor for local benchmark replace value task.
+ */
+ private static class BenchmarkReplaceValueEntryProcessor implements EntryProcessor<Object, Object, Object>, Serializable {
+ /**
+ * New value for update during process by default.
+ */
+ private Object newVal;
+
+ /**
+ * @param newVal default new value
+ */
+ private BenchmarkReplaceValueEntryProcessor(Object newVal) {
+ this.newVal = newVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object process(MutableEntry<Object, Object> entry, Object... arguments) {
+ Object newVal = arguments == null || arguments[0] == null ? this.newVal : arguments[0];
+
+ Object oldVal = entry.getValue();
+
+ entry.setValue(newVal);
+
+ return oldVal;
+ }
+ }
+
+ /**
+ * Entry processor for local benchmark remove entry task.
+ */
+ private static class BenchmarkRemoveEntryProcessor implements EntryProcessor<Object, Object, Object>, Serializable {
+ /** {@inheritDoc} */
+ @Override public Object process(MutableEntry<Object, Object> entry, Object... arguments) {
+ Object oldVal = entry.getValue();
+
+ entry.remove();
+
+ return oldVal;
+ }
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doInvokeAll(IgniteCache<Object, Object> cache) throws Exception {
+ Map<Object, EntryProcessor<Object, Object, Object>> map = new TreeMap<>();
+
+ Class keyCls = randomKeyClass(cache.getName());
+
+ for (int cnt = 0; cnt < args.batch(); cnt++) {
+ int i = nextRandom(args.range());
+
+ Object key = ModelUtil.create(keyCls, i);
+
+ if (nextBoolean())
+ map.put(key,
+ new BenchmarkReplaceValueEntryProcessor(createRandomValue(i + 1, cache.getName())));
+ else
+ map.put(key, rmvEntryProc);
+ }
+
+ cache.invokeAll(map);
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doRemove(IgniteCache<Object, Object> cache) throws Exception {
+ int i = nextRandom(args.range());
+
+ cache.remove(createRandomKey(i, cache.getName()));
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doRemoveAll(IgniteCache<Object, Object> cache) throws Exception {
+ Set<Object> keys = new TreeSet<>();
+
+ Class keyCls = randomKeyClass(cache.getName());
+
+ for (int cnt = 0; cnt < args.batch(); cnt++) {
+ int i = nextRandom(args.range());
+
+ keys.add(ModelUtil.create(keyCls, i));
+ }
+
+ cache.removeAll(keys);
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doPutIfAbsent(IgniteCache<Object, Object> cache) throws Exception {
+ int i = nextRandom(args.range());
+
+ cache.putIfAbsent(createRandomKey(i, cache.getName()), createRandomValue(i, cache.getName()));
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doReplace(IgniteCache<Object, Object> cache) throws Exception {
+ int i = nextRandom(args.range());
+
+ cache.replace(createRandomKey(i, cache.getName()),
+ createRandomValue(i, cache.getName()),
+ createRandomValue(i + 1, cache.getName()));
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doScanQuery(IgniteCache<Object, Object> cache) throws Exception {
+ if (!affCaches.contains(cache))
+ return;
+
+ Map<UUID, List<Integer>> partitionsMap = personCachePartitions(cache.getName());
+
+ ScanQueryBroadcastClosure c = new ScanQueryBroadcastClosure(cache.getName(), partitionsMap);
+
+ ClusterGroup clusterGrp = ignite().cluster().forNodeIds(partitionsMap.keySet());
+
+ IgniteCompute compute = ignite().compute(clusterGrp);
+
+ compute.broadcast(c);
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @throws Exception If failed.
+ */
+ private void doSqlQuery(IgniteCache<Object, Object> cache) throws Exception {
+ List<SqlCacheDescriptor> descriptors = cacheSqlDescriptors.get(cache.getName());
+
+ if (descriptors != null && !descriptors.isEmpty()) {
+ SqlCacheDescriptor randomDesc = descriptors.get(nextRandom(descriptors.size()));
+
+ int id = nextRandom(args.range());
+
+ Query sq;
+ if (queries.isEmpty())
+ sq = nextBoolean() ? randomDesc.getSqlQuery(id) : randomDesc.getSqlFieldsQuery(id);
+ else
+ sq = new SqlFieldsQuery(queries.get(nextRandom(queries.size())));
+
+ try (QueryCursor cursor = cache.query(sq)) {
+ for (Object obj : cursor) {
+ // No-op.
+ }
+ }
+ }
+ }
+
+ /**
+ * @param cache Ignite cache.
+ * @param map Parameters map.
+ * @throws Exception If failed.
+ */
+ private void doContinuousQuery(IgniteCache<Object, Object> cache, Map<Object, Object> map) throws Exception {
+ List<QueryCursor> cursors = (ArrayList<QueryCursor>)map.get(cache.getName());
+
+ if (cursors == null) {
+ cursors = new ArrayList<>(CONTINUOUS_QUERY_PER_CACHE);
+ map.put(cache.getName(), cursors);
+ }
+
+ if (cursors.size() == CONTINUOUS_QUERY_PER_CACHE) {
+ QueryCursor cursor = cursors.get(nextRandom(cursors.size()));
+ cursor.close();
+ cursors.remove(cursor);
+ }
+
+ ContinuousQuery qry = new ContinuousQuery();
+
+ qry.setLocalListener(new ContinuousQueryUpdater());
+
+ qry.setRemoteFilterFactory(FactoryBuilder.factoryOf(new ContinuousQueryFilter()));
+
+ cursors.add(cache.query(qry));
+ }
+
+ /**
+ * @return Nex random boolean value.
+ */
+ private boolean nextBoolean() {
+ return ThreadLocalRandom.current().nextBoolean();
+ }
+
+ /**
+ * Continuous query updater class.
+ */
+ private static class ContinuousQueryUpdater implements CacheEntryUpdatedListener, Serializable {
+
+ /** {@inheritDoc} */
+ @Override public void onUpdated(Iterable iterable) throws CacheEntryListenerException {
+ for (Object o : iterable) {
+ // No-op.
+ }
+ }
+ }
+
+ /**
+ * Continuous query filter class.
+ */
+ private static class ContinuousQueryFilter implements CacheEntryEventSerializableFilter, Serializable {
+ /** */
+ private boolean flag = true;
+
+ /** {@inheritDoc} */
+ @Override public boolean evaluate(CacheEntryEvent evt) throws CacheEntryListenerException {
+ return flag =! flag;
+ }
+ }
+
+ /**
+ * Closure for scan query executing.
+ */
+ private static class ScanQueryBroadcastClosure implements IgniteRunnable {
+ /**
+ * Ignite node.
+ */
+ @IgniteInstanceResource
+ private Ignite node;
+
+ /**
+ * Information about partition.
+ */
+ private Map<UUID, List<Integer>> cachePart;
+
+ /**
+ * Name of Ignite cache.
+ */
+ private String cacheName;
+
+ /**
+ * @param cacheName Name of Ignite cache.
+ * @param cachePart Partition by node for Ignite cache.
+ */
+ private ScanQueryBroadcastClosure(String cacheName, Map<UUID, List<Integer>> cachePart) {
+ this.cachePart = cachePart;
+ this.cacheName = cacheName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ IgniteCache cache = node.cache(cacheName);
+
+ // Getting a list of the partitions owned by this node.
+ List<Integer> myPartitions = cachePart.get(node.cluster().localNode().id());
+
+ for (Integer part : myPartitions) {
+
+ ScanQuery scanQry = new ScanQuery();
+
+ scanQry.setPartition(part);
+
+ scanQry.setFilter(igniteBiPred);
+
+ try (QueryCursor cursor = cache.query(scanQry)) {
+ for (Object obj : cursor) {
+ // No-op.
+ }
+ }
+
+ }
+ }
+ }
+
+ /**
+ * Scan query predicate class.
+ */
+ private static class BenchmarkIgniteBiPredicate implements IgniteBiPredicate {
+ /**
+ * @param key Cache key.
+ * @param val Cache value.
+ * @return true If is hit.
+ */
+ @Override public boolean apply(Object key, Object val) {
+ return true;
+ }
+ }
+
+ /**
+ * Query descriptor.
+ */
+ private static class SqlCacheDescriptor {
+ /**
+ * Class of value.
+ */
+ private Class valCls;
+
+ /**
+ * Select fields.
+ */
+ private Set<String> fields;
+
+ /**
+ * Indexed fields.
+ */
+ private Map<String, Class> indexedFieldsByCls;
+
+ /**
+ * @param valCls Class of value.
+ * @param fields All select fields.
+ * @param indexedFieldsByCls Indexed fields.
+ */
+ SqlCacheDescriptor(Class valCls, Set<String> fields,
+ Map<String, Class> indexedFieldsByCls) {
+ this.valCls = valCls;
+ this.fields = fields;
+ this.indexedFieldsByCls = indexedFieldsByCls;
+ }
+
+ /**
+ * @param id Query id.
+ * @return Condition string.
+ */
+ private String makeQuerySelect(int id) {
+ return StringUtils.collectionToDelimitedString(fields, ", ");
+ }
+
+ /**
+ * @param id Query id.
+ * @return Condition string.
+ */
+ private String makeQueryCondition(int id) {
+ StringBuilder sb = new StringBuilder();
+
+ for (String iField : indexedFieldsByCls.keySet()) {
+ Class cl = indexedFieldsByCls.get(iField);
+
+ if (!Number.class.isAssignableFrom(cl) && !String.class.equals(cl))
+ continue;
+
+ if (sb.length() != 0) {
+ switch (id % 3 % 2) {
+ case 0:
+ sb.append(" OR ");
+ break;
+ case 1:
+ sb.append(" AND ");
+ break;
+ }
+ }
+
+ if (Number.class.isAssignableFrom(cl)) {
+ sb.append(iField);
+ switch (id % 2) {
+ case 0:
+ sb.append(" > ");
+ break;
+ case 1:
+ sb.append(" < ");
+ break;
+ }
+ sb.append(id);
+ }
+ else if (String.class.equals(cl))
+ sb.append("lower(").append(iField).append(") LIKE lower('%").append(id).append("%')");
+
+ }
+ return sb.toString();
+ }
+
+ /**
+ * @param id Query id.
+ * @return SQL query object.
+ */
+ SqlQuery getSqlQuery(int id) {
+ return new SqlQuery(valCls, makeQueryCondition(id));
+ }
+
+ /**
+ * @param id Query id.
+ * @return SQL filed query object.
+ */
+ SqlFieldsQuery getSqlFieldsQuery(int id) {
+ return new SqlFieldsQuery(String.format("SELECT %s FROM %s WHERE %s",
+ makeQuerySelect(id), valCls.getSimpleName(), makeQueryCondition(id)));
+ }
+
+ }
+
+ /**
+ * Cache operation enum.
+ */
+ private enum Operation {
+ /** Put operation. */
+ PUT,
+
+ /** Put all operation. */
+ PUT_ALL,
+
+ /** Get operation. */
+ GET,
+
+ /** Get all operation. */
+ GET_ALL,
+
+ /** Invoke operation. */
+ INVOKE,
+
+ /** Invoke all operation. */
+ INVOKE_ALL,
+
+ /** Remove operation. */
+ REMOVE,
+
+ /** Remove all operation. */
+ REMOVE_ALL,
+
+ /** Put if absent operation. */
+ PUT_IF_ABSENT,
+
+ /** Replace operation. */
+ REPLACE,
+
+ /** Scan query operation. */
+ SCAN_QUERY,
+
+ /** SQL query operation. */
+ SQL_QUERY,
+
+ /** Continuous Query. */
+ CONTINUOUS_QUERY;
+
+ /**
+ * @param num Number of operation.
+ * @return Operation.
+ */
+ public static Operation valueOf(int num) {
+ return values()[num];
+ }
+ }
+}
[12/20] ignite git commit: Yardstick backport from master compilation
fix
Posted by vo...@apache.org.
Yardstick backport from master compilation fix
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6bf57cc1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6bf57cc1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6bf57cc1
Branch: refs/heads/ignite-3410
Commit: 6bf57cc172434441b3359b64c6128451e22cd9c1
Parents: 4670bc5
Author: Anton Vinogradov <av...@apache.org>
Authored: Wed Jun 29 16:23:08 2016 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed Jun 29 16:23:08 2016 +0300
----------------------------------------------------------------------
modules/yardstick/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf57cc1/modules/yardstick/pom.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index a0f7de5..52418d9 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -31,7 +31,7 @@
</parent>
<artifactId>ignite-yardstick</artifactId>
- <version>1.7.0-SNAPSHOT</version>
+ <version>1.6.0-SNAPSHOT</version>
<url>http://ignite.apache.org</url>
<properties>
[06/20] ignite git commit: IGNITE-3383: Hadoop: Improved Hadoop
component start logic.
Posted by vo...@apache.org.
IGNITE-3383: Hadoop: Improved Hadoop component start logic.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bdeef4c0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bdeef4c0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bdeef4c0
Branch: refs/heads/ignite-3410
Commit: bdeef4c06a807ca9a4e7f2ed4faa4ca3f0d26f20
Parents: a77e690
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jun 28 16:33:09 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jun 28 16:33:09 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/IgniteKernal.java | 48 ++++++++++++------
.../processors/hadoop/HadoopNoopProcessor.java | 5 ++
.../hadoop/HadoopProcessorAdapter.java | 7 +++
.../processors/hadoop/HadoopClassLoader.java | 2 +-
.../processors/hadoop/HadoopProcessor.java | 52 +++++++++-----------
5 files changed, 68 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 3572293..b6ba4cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -114,7 +114,7 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
import org.apache.ignite.internal.processors.hadoop.Hadoop;
-import org.apache.ignite.internal.processors.hadoop.HadoopNoopProcessor;
+import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
import org.apache.ignite.internal.processors.job.GridJobProcessor;
import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
import org.apache.ignite.internal.processors.nodevalidation.DiscoveryNodeValidationProcessor;
@@ -1087,26 +1087,44 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
* Create Hadoop component.
*
* @return Non-null Hadoop component: workable or no-op.
- * @throws IgniteCheckedException if the component is mandatory and cannot be initialized.
+ * @throws IgniteCheckedException If the component is mandatory and cannot be initialized.
*/
- private GridComponent createHadoopComponent() throws IgniteCheckedException {
- GridComponent cmp;
+ private HadoopProcessorAdapter createHadoopComponent() throws IgniteCheckedException {
+ boolean mandatory = cfg.getHadoopConfiguration() != null;
- if (cfg.isPeerClassLoadingEnabled()) {
- cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false);
+ if (mandatory) {
+ if (cfg.isPeerClassLoadingEnabled())
+ throw new IgniteCheckedException("Hadoop module cannot be used with peer class loading enabled " +
+ "(set IgniteConfiguration.peerClassLoadingEnabled to \"false\").");
- if (!(cmp instanceof HadoopNoopProcessor)) {
- U.warn(log, "Hadoop module is found in classpath, but it will not be started because peer class " +
- "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled to \"false\" to start " +
- "Hadoop module).");
+ return IgniteComponentType.HADOOP.createIfInClassPath(ctx, true);
+ }
+ else {
+ HadoopProcessorAdapter cmp = null;
- cmp = IgniteComponentType.HADOOP.create(ctx, true/*no-op*/);
+ if (IgniteComponentType.HADOOP.inClassPath() && cfg.isPeerClassLoadingEnabled()) {
+ U.warn(log, "Hadoop module is found in classpath, but will not be started because peer class " +
+ "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled to \"false\" if you want " +
+ "to use Hadoop module).");
}
- }
- else
- cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, cfg.getHadoopConfiguration() != null);
+ else {
+ cmp = IgniteComponentType.HADOOP.createIfInClassPath(ctx, false);
+
+ try {
+ cmp.validateEnvironment();
+ }
+ catch (IgniteException | IgniteCheckedException e) {
+ U.quietAndWarn(log, "Hadoop module will not start due to exception: " + e.getMessage());
+
+ cmp = null;
+ }
+ }
+
+ if (cmp == null)
+ cmp = IgniteComponentType.HADOOP.create(ctx, true);
- return cmp;
+ return cmp;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
index 72b5050..065cd72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
@@ -74,4 +74,9 @@ public class HadoopNoopProcessor extends HadoopProcessorAdapter {
@Override public boolean kill(HadoopJobId jobId) throws IgniteCheckedException {
return false;
}
+
+ /** {@inheritDoc} */
+ @Override public void validateEnvironment() throws IgniteCheckedException {
+ // No-oop.
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
index 3d381c3..f914639 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessorAdapter.java
@@ -94,4 +94,11 @@ public abstract class HadoopProcessorAdapter extends GridProcessorAdapter {
* @throws IgniteCheckedException If failed.
*/
public abstract boolean kill(HadoopJobId jobId) throws IgniteCheckedException;
+
+ /**
+ * Validate Hadoop environment.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void validateEnvironment() throws IgniteCheckedException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 1c844c4..389de8c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -501,7 +501,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
hadoopUrls = HadoopClasspathUtils.classpathUrls();
}
catch (IOException e) {
- throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations.", e);
+ throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations: " + e.getMessage(), e);
}
hadoopJars = hadoopUrls;
http://git-wip-us.apache.org/repos/asf/ignite/blob/bdeef4c0/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
index c45953e..7fc7499 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java
@@ -50,6 +50,8 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
private Hadoop hadoop;
/**
+ * Constructor.
+ *
* @param ctx Kernal context.
*/
public HadoopProcessor(GridKernalContext ctx) {
@@ -70,24 +72,6 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
initializeDefaults(cfg);
- validate(cfg);
-
- try {
- HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
-
- if (loc.home() != null)
- U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
-
- U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
- U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
- U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
- }
- catch (IOException ioe) {
- throw new IgniteCheckedException(ioe);
- }
-
- HadoopClassLoader.hadoopUrls();
-
hctx = new HadoopContext(
ctx,
cfg,
@@ -204,6 +188,26 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
return hctx.jobTracker().killJob(jobId);
}
+ /** {@inheritDoc} */
+ @Override public void validateEnvironment() throws IgniteCheckedException {
+ // Perform some static checks as early as possible, so that any recoverable exceptions are thrown here.
+ try {
+ HadoopLocations loc = HadoopClasspathUtils.hadoopLocations();
+
+ if (loc.home() != null)
+ U.quietAndInfo(log, "HADOOP_HOME is set to " + loc.home());
+
+ U.quietAndInfo(log, "HADOOP_COMMON_HOME is set to " + loc.commonHome());
+ U.quietAndInfo(log, "HADOOP_HDFS_HOME is set to " + loc.hdfsHome());
+ U.quietAndInfo(log, "HADOOP_MAPRED_HOME is set to " + loc.mapredHome());
+ }
+ catch (IOException ioe) {
+ throw new IgniteCheckedException(ioe.getMessage(), ioe);
+ }
+
+ HadoopClassLoader.hadoopUrls();
+ }
+
/**
* Initializes default hadoop configuration.
*
@@ -213,16 +217,4 @@ public class HadoopProcessor extends HadoopProcessorAdapter {
if (cfg.getMapReducePlanner() == null)
cfg.setMapReducePlanner(new IgniteHadoopMapReducePlanner());
}
-
- /**
- * Validates Grid and Hadoop configuration for correctness.
- *
- * @param hadoopCfg Hadoop configuration.
- * @throws IgniteCheckedException If failed.
- */
- private void validate(HadoopConfiguration hadoopCfg) throws IgniteCheckedException {
- if (ctx.config().isPeerClassLoadingEnabled())
- throw new IgniteCheckedException("Peer class loading cannot be used with Hadoop (disable it using " +
- "IgniteConfiguration.setPeerClassLoadingEnabled()).");
- }
}
\ No newline at end of file
[18/20] ignite git commit: Revert "IGNITE-3154: Optimized BinaryField
lookup within a BinaryType."
Posted by vo...@apache.org.
Revert "IGNITE-3154: Optimized BinaryField lookup within a BinaryType."
This reverts commit 62c714340ef6c40de4dcdb6d9ecaf11ab30df834.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6961e637
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6961e637
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6961e637
Branch: refs/heads/ignite-3410
Commit: 6961e6375e5587b6a0dc24c5ce16338ee6a407f8
Parents: 02d395c
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 30 09:27:08 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 30 09:27:08 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/binary/BinaryContext.java | 1 +
.../internal/binary/BinaryEnumObjectImpl.java | 2 +-
.../internal/binary/BinaryObjectImpl.java | 5 +-
.../binary/BinaryObjectOffheapImpl.java | 5 +-
.../ignite/internal/binary/BinaryTypeProxy.java | 107 -------------------
.../ignite/internal/binary/BinaryUtils.java | 21 +---
6 files changed, 12 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6961e637/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 3853841..d39f9c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -83,6 +83,7 @@ import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaFileUnlockProcess
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaUpdatePropertiesProcessor;
import org.apache.ignite.internal.processors.igfs.meta.IgfsMetaUpdateTimesProcessor;
import org.apache.ignite.internal.processors.platform.PlatformJavaObjectFactoryProxy;
+import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.lang.GridMapEntry;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T2;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6961e637/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
index eea85ef9..60642ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
@@ -131,7 +131,7 @@ public class BinaryEnumObjectImpl implements BinaryObjectEx, Externalizable, Cac
/** {@inheritDoc} */
@Override public BinaryType type() throws BinaryObjectException {
- return BinaryUtils.typeProxy(ctx, this);
+ return ctx.metadata(typeId());
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6961e637/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
index 9ebee0b..173bb6e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
@@ -227,7 +227,10 @@ public final class BinaryObjectImpl extends BinaryObjectExImpl implements Extern
/** {@inheritDoc} */
@Nullable @Override public BinaryType type() throws BinaryObjectException {
- return BinaryUtils.typeProxy(ctx, this);
+ if (ctx == null)
+ throw new BinaryObjectException("BinaryContext is not set for the object.");
+
+ return ctx.metadata(typeId());
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6961e637/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
index 60527d1..509247f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java
@@ -142,7 +142,10 @@ public class BinaryObjectOffheapImpl extends BinaryObjectExImpl implements Exter
/** {@inheritDoc} */
@Nullable @Override public BinaryType type() throws BinaryObjectException {
- return BinaryUtils.typeProxy(ctx, this);
+ if (ctx == null)
+ throw new BinaryObjectException("BinaryContext is not set for the object.");
+
+ return ctx.metadata(typeId());
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6961e637/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
deleted file mode 100644
index e10f2f7..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryTypeProxy.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.binary;
-
-import org.apache.ignite.binary.BinaryField;
-import org.apache.ignite.binary.BinaryType;
-import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-
-import java.util.Collection;
-
-/**
- * Binary type proxy. Is used to delay or completely avoid metadata lookup.
- */
-public class BinaryTypeProxy implements BinaryType {
- /** Binary context. */
- @GridToStringExclude
- private final BinaryContext ctx;
-
- /** Type ID. */
- private final int typeId;
-
- /** Target type. */
- @GridToStringExclude
- private volatile BinaryType target;
-
- /**
- * Constrcutor.
- *
- * @param ctx Context.
- * @param typeId Type ID.
- */
- public BinaryTypeProxy(BinaryContext ctx, int typeId) {
- this.ctx = ctx;
- this.typeId = typeId;
- }
-
- /** {@inheritDoc} */
- @Override public int typeId() {
- return typeId;
- }
-
- /** {@inheritDoc} */
- @Override public BinaryField field(String fieldName) {
- return ctx.createField(typeId, fieldName);
- }
-
- /** {@inheritDoc} */
- @Override public String typeName() {
- return target().typeName();
- }
-
- /** {@inheritDoc} */
- @Override public Collection<String> fieldNames() {
- return target().fieldNames();
- }
-
- /** {@inheritDoc} */
- @Override public String fieldTypeName(String fieldName) {
- return target().fieldTypeName(fieldName);
- }
-
- /** {@inheritDoc} */
- @Override public String affinityKeyFieldName() {
- return target().affinityKeyFieldName();
- }
-
- /** {@inheritDoc} */
- @Override public boolean isEnum() {
- return target().isEnum();
- }
-
- /**
- * @return Target ype.
- */
- private BinaryType target() {
- if (target == null) {
- synchronized (this) {
- if (target == null) {
- target = ctx.metadata(typeId);
- }
- }
- }
-
- return target;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(BinaryTypeProxy.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6961e637/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index e986228..0250e3b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -54,7 +54,6 @@ import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.binary.BinaryRawReader;
import org.apache.ignite.binary.BinaryRawWriter;
-import org.apache.ignite.binary.BinaryType;
import org.apache.ignite.binary.Binarylizable;
import org.apache.ignite.internal.binary.builder.BinaryLazyValue;
import org.apache.ignite.internal.binary.streams.BinaryInputStream;
@@ -603,7 +602,6 @@ public class BinaryUtils {
* @param map Map.
* @return New map of the same type or null.
*/
- @SuppressWarnings("unchecked")
public static <K, V> Map<K, V> newKnownMap(Object map) {
Class<?> cls = map == null ? null : map.getClass();
@@ -678,7 +676,6 @@ public class BinaryUtils {
* @param col Collection.
* @return New empty collection.
*/
- @SuppressWarnings("unchecked")
public static <V> Collection<V> newKnownCollection(Object col) {
Class<?> cls = col == null ? null : col.getClass();
@@ -1395,7 +1392,6 @@ public class BinaryUtils {
/**
* @return Value.
*/
- @SuppressWarnings("ConstantConditions")
public static Object doReadProxy(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr,
BinaryReaderHandlesHolder handles) {
Class<?>[] intfs = new Class<?>[in.readInt()];
@@ -2127,7 +2123,8 @@ public class BinaryUtils {
throw new BinaryObjectException("Malformed input around byte: " + (off - 1));
res[charArrCnt++] = (char)(((c & 0x0F) << 12) |
- ((c2 & 0x3F) << 6) | (c3 & 0x3F));
+ ((c2 & 0x3F) << 6) |
+ ((c3 & 0x3F) << 0));
break;
default:
@@ -2186,20 +2183,6 @@ public class BinaryUtils {
}
/**
- * Create binary type proxy.
- *
- * @param ctx Context.
- * @param obj Binary object.
- * @return Binary type proxy.
- */
- public static BinaryType typeProxy(BinaryContext ctx, BinaryObjectEx obj) {
- if (ctx == null)
- throw new BinaryObjectException("BinaryContext is not set for the object.");
-
- return new BinaryTypeProxy(ctx, obj.typeId());
- }
-
- /**
* Enum type.
*/
private static class EnumType {