You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by se...@apache.org on 2015/02/02 22:57:31 UTC
[48/50] [abbrv] incubator-ignite git commit: ignite-132 - pkg rename
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/hadoop/GridHadoopDefaultJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/GridHadoopDefaultJobInfo.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/GridHadoopDefaultJobInfo.java
deleted file mode 100644
index 8e5e7ab..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/GridHadoopDefaultJobInfo.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.hadoop;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.processors.hadoop.*;
-import org.apache.ignite.internal.processors.hadoop.v2.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.lang.reflect.*;
-import java.util.*;
-
-/**
- * Hadoop job info based on default Hadoop configuration.
- */
-public class GridHadoopDefaultJobInfo implements GridHadoopJobInfo, Externalizable {
- /** */
- private static final long serialVersionUID = 5489900236464999951L;
-
- /** {@code true} If job has combiner. */
- private boolean hasCombiner;
-
- /** Number of reducers configured for job. */
- private int numReduces;
-
- /** Configuration. */
- private Map<String,String> props = new HashMap<>();
-
- /** Job name. */
- private String jobName;
-
- /** User name. */
- private String user;
-
- /** */
- private static volatile Class<?> jobCls;
-
- /**
- * Default constructor required by {@link Externalizable}.
- */
- public GridHadoopDefaultJobInfo() {
- // No-op.
- }
-
- /**
- * Constructor.
- *
- * @param jobName Job name.
- * @param user User name.
- * @param hasCombiner {@code true} If job has combiner.
- * @param numReduces Number of reducers configured for job.
- * @param props All other properties of the job.
- */
- public GridHadoopDefaultJobInfo(String jobName, String user, boolean hasCombiner, int numReduces,
- Map<String, String> props) {
- this.jobName = jobName;
- this.user = user;
- this.hasCombiner = hasCombiner;
- this.numReduces = numReduces;
- this.props = props;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public String property(String name) {
- return props.get(name);
- }
-
- /** {@inheritDoc} */
- @Override public GridHadoopJob createJob(GridHadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException {
- try {
- Class<?> jobCls0 = jobCls;
-
- if (jobCls0 == null) { // It is enough to have only one class loader with only Hadoop classes.
- synchronized (GridHadoopDefaultJobInfo.class) {
- if ((jobCls0 = jobCls) == null) {
- GridHadoopClassLoader ldr = new GridHadoopClassLoader(null);
-
- jobCls = jobCls0 = ldr.loadClass(GridHadoopV2Job.class.getName());
- }
- }
- }
-
- Constructor<?> constructor = jobCls0.getConstructor(GridHadoopJobId.class, GridHadoopDefaultJobInfo.class,
- IgniteLogger.class);
-
- return (GridHadoopJob)constructor.newInstance(jobId, this, log);
- }
- catch (Exception e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasCombiner() {
- return hasCombiner;
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasReducer() {
- return reducers() > 0;
- }
-
- /** {@inheritDoc} */
- @Override public int reducers() {
- return numReduces;
- }
-
- /** {@inheritDoc} */
- @Override public String jobName() {
- return jobName;
- }
-
- /** {@inheritDoc} */
- @Override public String user() {
- return user;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- U.writeString(out, jobName);
- U.writeString(out, user);
-
- out.writeBoolean(hasCombiner);
- out.writeInt(numReduces);
-
- U.writeStringMap(out, props);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobName = U.readString(in);
- user = U.readString(in);
-
- hasCombiner = in.readBoolean();
- numReduces = in.readInt();
-
- props = U.readStringMap(in);
- }
-
- /**
- * @return Properties of the job.
- */
- public Map<String, String> properties() {
- return props;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/hadoop/GridHadoopSetup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/GridHadoopSetup.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/GridHadoopSetup.java
deleted file mode 100644
index 0fbfdf7..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/GridHadoopSetup.java
+++ /dev/null
@@ -1,506 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.hadoop;
-
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-import java.net.*;
-import java.nio.file.*;
-import java.text.*;
-import java.util.*;
-
-import static org.apache.ignite.internal.GridProductImpl.*;
-
-/**
- * Setup tool to configure Hadoop client.
- */
-public class GridHadoopSetup {
- /** */
- public static final String WINUTILS_EXE = "winutils.exe";
-
- /** */
- private static final FilenameFilter IGNITE_JARS = new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return name.startsWith("ignite-") && name.endsWith(".jar");
- }
- };
-
- /**
- * The main method.
- * @param ignore Params.
- * @throws IOException If fails.
- */
- public static void main(String[] ignore) throws IOException {
- X.println(
- " __________ ________________ ",
- " / _/ ___/ |/ / _/_ __/ __/ ",
- " _/ // (_ / // / / / / _/ ",
- "/___/\\___/_/|_/___/ /_/ /___/ ",
- " for Apache Hadoop ",
- " ");
-
- println("Version " + ACK_VER);
-
- configureHadoop();
- }
-
- /**
- * This operation prepares the clean unpacked Hadoop distributive to work as client with GridGain-Hadoop.
- * It performs these operations:
- * <ul>
- * <li>Check for setting of HADOOP_HOME environment variable.</li>
- * <li>Try to resolve HADOOP_COMMON_HOME or evaluate it relative to HADOOP_HOME.</li>
- * <li>In Windows check if winutils.exe exists and try to fix issue with some restrictions.</li>
- * <li>In Windows check new line character issues in CMD scripts.</li>
- * <li>Scan Hadoop lib directory to detect GridGain JARs. If these don't exist tries to create ones.</li>
- * </ul>
- */
- private static void configureHadoop() {
- String gridgainHome = U.getGridGainHome();
-
- println("IGNITE_HOME is set to '" + gridgainHome + "'.");
-
- checkGridGainHome(gridgainHome);
-
- String homeVar = "HADOOP_HOME";
- String hadoopHome = System.getenv(homeVar);
-
- if (F.isEmpty(hadoopHome)) {
- homeVar = "HADOOP_PREFIX";
- hadoopHome = System.getenv(homeVar);
- }
-
- if (F.isEmpty(hadoopHome))
- exit("Neither HADOOP_HOME nor HADOOP_PREFIX environment variable is set. Please set one of them to a " +
- "valid Hadoop installation directory and run setup tool again.", null);
-
- hadoopHome = hadoopHome.replaceAll("\"", "");
-
- println(homeVar + " is set to '" + hadoopHome + "'.");
-
- String hiveHome = System.getenv("HIVE_HOME");
-
- if (!F.isEmpty(hiveHome)) {
- hiveHome = hiveHome.replaceAll("\"", "");
-
- println("HIVE_HOME is set to '" + hiveHome + "'.");
- }
-
- File hadoopDir = new File(hadoopHome);
-
- if (!hadoopDir.exists())
- exit("Hadoop installation folder does not exist.", null);
-
- if (!hadoopDir.isDirectory())
- exit("HADOOP_HOME must point to a directory.", null);
-
- if (!hadoopDir.canRead())
- exit("Hadoop installation folder can not be read. Please check permissions.", null);
-
- File hadoopCommonDir;
-
- String hadoopCommonHome = System.getenv("HADOOP_COMMON_HOME");
-
- if (F.isEmpty(hadoopCommonHome)) {
- hadoopCommonDir = new File(hadoopDir, "share/hadoop/common");
-
- println("HADOOP_COMMON_HOME is not set, will use '" + hadoopCommonDir.getPath() + "'.");
- }
- else {
- println("HADOOP_COMMON_HOME is set to '" + hadoopCommonHome + "'.");
-
- hadoopCommonDir = new File(hadoopCommonHome);
- }
-
- if (!hadoopCommonDir.canRead())
- exit("Failed to read Hadoop common dir in '" + hadoopCommonHome + "'.", null);
-
- File hadoopCommonLibDir = new File(hadoopCommonDir, "lib");
-
- if (!hadoopCommonLibDir.canRead())
- exit("Failed to read Hadoop 'lib' folder in '" + hadoopCommonLibDir.getPath() + "'.", null);
-
- if (U.isWindows()) {
- checkJavaPathSpaces();
-
- File hadoopBinDir = new File(hadoopDir, "bin");
-
- if (!hadoopBinDir.canRead())
- exit("Failed to read subdirectory 'bin' in HADOOP_HOME.", null);
-
- File winutilsFile = new File(hadoopBinDir, WINUTILS_EXE);
-
- if (!winutilsFile.exists()) {
- if (ask("File '" + WINUTILS_EXE + "' does not exist. " +
- "It may be replaced by a stub. Create it?")) {
- println("Creating file stub '" + winutilsFile.getAbsolutePath() + "'.");
-
- boolean ok = false;
-
- try {
- ok = winutilsFile.createNewFile();
- }
- catch (IOException ignore) {
- // No-op.
- }
-
- if (!ok)
- exit("Failed to create '" + WINUTILS_EXE + "' file. Please check permissions.", null);
- }
- else
- println("Ok. But Hadoop client probably will not work on Windows this way...");
- }
-
- processCmdFiles(hadoopDir, "bin", "sbin", "libexec");
- }
-
- File gridgainLibs = new File(new File(gridgainHome), "libs");
-
- if (!gridgainLibs.exists())
- exit("GridGain 'libs' folder is not found.", null);
-
- Collection<File> jarFiles = new ArrayList<>();
-
- addJarsInFolder(jarFiles, gridgainLibs);
- addJarsInFolder(jarFiles, new File(gridgainLibs, "gridgain-hadoop"));
-
- boolean jarsLinksCorrect = true;
-
- for (File file : jarFiles) {
- File link = new File(hadoopCommonLibDir, file.getName());
-
- jarsLinksCorrect &= isJarLinkCorrect(link, file);
-
- if (!jarsLinksCorrect)
- break;
- }
-
- if (!jarsLinksCorrect) {
- if (ask("GridGain JAR files are not found in Hadoop 'lib' directory. " +
- "Create appropriate symbolic links?")) {
- File[] oldGridGainJarFiles = hadoopCommonLibDir.listFiles(IGNITE_JARS);
-
- if (oldGridGainJarFiles.length > 0 && ask("The Hadoop 'lib' directory contains JARs from other GridGain " +
- "installation. They must be deleted to continue. Continue?")) {
- for (File file : oldGridGainJarFiles) {
- println("Deleting file '" + file.getAbsolutePath() + "'.");
-
- if (!file.delete())
- exit("Failed to delete file '" + file.getPath() + "'.", null);
- }
- }
-
- for (File file : jarFiles) {
- File targetFile = new File(hadoopCommonLibDir, file.getName());
-
- try {
- println("Creating symbolic link '" + targetFile.getAbsolutePath() + "'.");
-
- Files.createSymbolicLink(targetFile.toPath(), file.toPath());
- }
- catch (IOException e) {
- if (U.isWindows()) {
- warn("Ability to create symbolic links is required!");
- warn("On Windows platform you have to grant permission 'Create symbolic links'");
- warn("to your user or run the Accelerator as Administrator.");
- }
-
- exit("Creating symbolic link failed! Check permissions.", e);
- }
- }
- }
- else
- println("Ok. But Hadoop client will not be able to talk to GridGain cluster without those JARs in classpath...");
- }
-
- File hadoopEtc = new File(hadoopDir, "etc" + File.separator + "hadoop");
-
- File gridgainDocs = new File(gridgainHome, "docs");
-
- if (!gridgainDocs.canRead())
- exit("Failed to read GridGain 'docs' folder at '" + gridgainDocs.getAbsolutePath() + "'.", null);
-
- if (hadoopEtc.canWrite()) { // TODO Bigtop
- if (ask("Replace 'core-site.xml' and 'mapred-site.xml' files with preconfigured templates " +
- "(existing files will be backed up)?")) {
- replaceWithBackup(new File(gridgainDocs, "core-site.gridgain.xml"), new File(hadoopEtc, "core-site.xml"));
-
- replaceWithBackup(new File(gridgainDocs, "mapred-site.gridgain.xml"), new File(hadoopEtc, "mapred-site.xml"));
- }
- else
- println("Ok. You can configure them later, the templates are available at GridGain's 'docs' directory...");
- }
-
- if (!F.isEmpty(hiveHome)) {
- File hiveConfDir = new File(hiveHome + File.separator + "conf");
-
- if (!hiveConfDir.canWrite())
- warn("Can not write to '" + hiveConfDir.getAbsolutePath() + "'. To run Hive queries you have to " +
- "configure 'hive-site.xml' manually. The template is available at GridGain's 'docs' directory.");
- else if (ask("Replace 'hive-site.xml' with preconfigured template (existing file will be backed up)?"))
- replaceWithBackup(new File(gridgainDocs, "hive-site.gridgain.xml"), new File(hiveConfDir, "hive-site.xml"));
- else
- println("Ok. You can configure it later, the template is available at GridGain's 'docs' directory...");
- }
-
- println("Apache Hadoop setup is complete.");
- }
-
- /**
- * @param jarFiles Jars.
- * @param folder Folder.
- */
- private static void addJarsInFolder(Collection<File> jarFiles, File folder) {
- if (!folder.exists())
- exit("Folder '" + folder.getAbsolutePath() + "' is not found.", null);
-
- jarFiles.addAll(Arrays.asList(folder.listFiles(IGNITE_JARS)));
- }
-
- /**
- * Checks that JAVA_HOME does not contain space characters.
- */
- private static void checkJavaPathSpaces() {
- String javaHome = System.getProperty("java.home");
-
- if (javaHome.contains(" ")) {
- warn("Java installation path contains space characters!");
- warn("Hadoop client will not be able to start using '" + javaHome + "'.");
- warn("Please install JRE to path which does not contain spaces and point JAVA_HOME to that installation.");
- }
- }
-
- /**
- * Checks GridGain home.
- *
- * @param ggHome GridGain home.
- */
- private static void checkGridGainHome(String ggHome) {
- URL jarUrl = U.class.getProtectionDomain().getCodeSource().getLocation();
-
- try {
- Path jar = Paths.get(jarUrl.toURI());
- Path gg = Paths.get(ggHome);
-
- if (!jar.startsWith(gg))
- exit("GridGain JAR files are not under IGNITE_HOME.", null);
- }
- catch (Exception e) {
- exit(e.getMessage(), e);
- }
- }
-
- /**
- * Replaces target file with source file.
- *
- * @param from From.
- * @param to To.
- */
- private static void replaceWithBackup(File from, File to) {
- if (!from.canRead())
- exit("Failed to read source file '" + from.getAbsolutePath() + "'.", null);
-
- println("Replacing file '" + to.getAbsolutePath() + "'.");
-
- try {
- U.copy(from, renameToBak(to), true);
- }
- catch (IOException e) {
- exit("Failed to replace file '" + to.getAbsolutePath() + "'.", e);
- }
- }
-
- /**
- * Renames file for backup.
- *
- * @param file File.
- * @return File.
- */
- private static File renameToBak(File file) {
- DateFormat fmt = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
-
- if (file.exists() && !file.renameTo(new File(file.getAbsolutePath() + "." + fmt.format(new Date()) + ".bak")))
- exit("Failed to rename file '" + file.getPath() + "'.", null);
-
- return file;
- }
-
- /**
- * Checks if link is correct.
- *
- * @param link Symbolic link.
- * @param correctTarget Correct link target.
- * @return {@code true} If link target is correct.
- */
- private static boolean isJarLinkCorrect(File link, File correctTarget) {
- if (!Files.isSymbolicLink(link.toPath()))
- return false; // It is a real file or it does not exist.
-
- Path target = null;
-
- try {
- target = Files.readSymbolicLink(link.toPath());
- }
- catch (IOException e) {
- exit("Failed to read symbolic link: " + link.getAbsolutePath(), e);
- }
-
- return Files.exists(target) && target.toFile().equals(correctTarget);
- }
-
- /**
- * Writes the question end read the boolean answer from the console.
- *
- * @param question Question to write.
- * @return {@code true} if user inputs 'Y' or 'y', {@code false} otherwise.
- */
- private static boolean ask(String question) {
- X.println();
- X.print(" < " + question + " (Y/N): ");
-
- String answer = null;
-
- if (!F.isEmpty(System.getenv("IGNITE_HADOOP_SETUP_YES")))
- answer = "Y";
- else {
- BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
-
- try {
- answer = br.readLine();
- }
- catch (IOException e) {
- exit("Failed to read answer: " + e.getMessage(), e);
- }
- }
-
- if (answer != null && "Y".equals(answer.toUpperCase().trim())) {
- X.println(" > Yes.");
-
- return true;
- }
- else {
- X.println(" > No.");
-
- return false;
- }
- }
-
- /**
- * Exit with message.
- *
- * @param msg Exit message.
- */
- private static void exit(String msg, Exception e) {
- X.println(" ");
- X.println(" # " + msg);
- X.println(" # Setup failed, exiting... ");
-
- if (e != null && !F.isEmpty(System.getenv("IGNITE_HADOOP_SETUP_DEBUG")))
- e.printStackTrace();
-
- System.exit(1);
- }
-
- /**
- * Prints message.
- *
- * @param msg Message.
- */
- private static void println(String msg) {
- X.println(" > " + msg);
- }
-
- /**
- * Prints warning.
- *
- * @param msg Message.
- */
- private static void warn(String msg) {
- X.println(" ! " + msg);
- }
-
- /**
- * Checks that CMD files have valid MS Windows new line characters. If not, writes question to console and reads the
- * answer. If it's 'Y' then backups original files and corrects invalid new line characters.
- *
- * @param rootDir Root directory to process.
- * @param dirs Directories inside of the root to process.
- */
- private static void processCmdFiles(File rootDir, String... dirs) {
- boolean answer = false;
-
- for (String dir : dirs) {
- File subDir = new File(rootDir, dir);
-
- File[] cmdFiles = subDir.listFiles(new FilenameFilter() {
- @Override public boolean accept(File dir, String name) {
- return name.toLowerCase().endsWith(".cmd");
- }
- });
-
- for (File file : cmdFiles) {
- String content = null;
-
- try (Scanner scanner = new Scanner(file)) {
- content = scanner.useDelimiter("\\Z").next();
- }
- catch (FileNotFoundException e) {
- exit("Failed to read file '" + file + "'.", e);
- }
-
- boolean invalid = false;
-
- for (int i = 0; i < content.length(); i++) {
- if (content.charAt(i) == '\n' && (i == 0 || content.charAt(i - 1) != '\r')) {
- invalid = true;
-
- break;
- }
- }
-
- if (invalid) {
- answer = answer || ask("One or more *.CMD files has invalid new line character. Replace them?");
-
- if (!answer) {
- println("Ok. But Windows most probably will fail to execute them...");
-
- return;
- }
-
- println("Fixing newline characters in file '" + file.getAbsolutePath() + "'.");
-
- renameToBak(file);
-
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) {
- for (int i = 0; i < content.length(); i++) {
- if (content.charAt(i) == '\n' && (i == 0 || content.charAt(i - 1) != '\r'))
- writer.write("\r");
-
- writer.write(content.charAt(i));
- }
- }
- catch (IOException e) {
- exit("Failed to write file '" + file.getPath() + "': " + e.getMessage(), e);
- }
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopEndpoint.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopEndpoint.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopEndpoint.java
index 269e150..ce0c867 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopEndpoint.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopEndpoint.java
@@ -18,9 +18,9 @@
package org.apache.ignite.internal.fs.hadoop;
import org.apache.ignite.*;
-import org.apache.ignite.lang.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
import org.jetbrains.annotations.*;
import java.io.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopFileSystemWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopFileSystemWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopFileSystemWrapper.java
index 5a107a1..47b6e15 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopFileSystemWrapper.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopFileSystemWrapper.java
@@ -19,20 +19,20 @@ package org.apache.ignite.internal.fs.hadoop;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.*;
import org.apache.hadoop.ipc.*;
import org.apache.ignite.*;
import org.apache.ignite.fs.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-
import org.apache.ignite.internal.processors.fs.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.jetbrains.annotations.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
/**
* Adapter to use any Hadoop file system {@link org.apache.hadoop.fs.FileSystem} as {@link org.apache.ignite.fs.IgniteFsFileSystem}.
*/
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopIpcIo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopIpcIo.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopIpcIo.java
index 44b8a5b..40d26c6 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopIpcIo.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopIpcIo.java
@@ -20,8 +20,8 @@ package org.apache.ignite.internal.fs.hadoop;
import org.apache.commons.logging.*;
import org.apache.ignite.*;
import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.fs.common.*;
+import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.ipc.*;
import org.apache.ignite.internal.util.ipc.shmem.*;
import org.apache.ignite.internal.util.lang.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopReader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopReader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopReader.java
index 297ca5d..64acb2e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopReader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/fs/hadoop/GridGgfsHadoopReader.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.fs.hadoop;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.ignite.fs.*;
import org.apache.ignite.internal.util.typedef.internal.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopContext.java
index 8038abf..3160e3d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopContext.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopContext.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.hadoop;
import org.apache.ignite.cluster.*;
-import org.apache.ignite.hadoop.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
import org.apache.ignite.internal.processors.hadoop.shuffle.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultJobInfo.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultJobInfo.java
new file mode 100644
index 0000000..87db9c8
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopDefaultJobInfo.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.hadoop.v2.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.lang.reflect.*;
+import java.util.*;
+
+/**
+ * Hadoop job info based on default Hadoop configuration.
+ */
+public class GridHadoopDefaultJobInfo implements GridHadoopJobInfo, Externalizable {
+ /** */
+ private static final long serialVersionUID = 5489900236464999951L;
+
+ /** {@code true} If job has combiner. */
+ private boolean hasCombiner;
+
+ /** Number of reducers configured for job. */
+ private int numReduces;
+
+ /** Configuration. */
+ private Map<String,String> props = new HashMap<>();
+
+ /** Job name. */
+ private String jobName;
+
+ /** User name. */
+ private String user;
+
+ /** */
+ private static volatile Class<?> jobCls;
+
+ /**
+ * Default constructor required by {@link Externalizable}.
+ */
+ public GridHadoopDefaultJobInfo() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param jobName Job name.
+ * @param user User name.
+ * @param hasCombiner {@code true} If job has combiner.
+ * @param numReduces Number of reducers configured for job.
+ * @param props All other properties of the job.
+ */
+ public GridHadoopDefaultJobInfo(String jobName, String user, boolean hasCombiner, int numReduces,
+ Map<String, String> props) {
+ this.jobName = jobName;
+ this.user = user;
+ this.hasCombiner = hasCombiner;
+ this.numReduces = numReduces;
+ this.props = props;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String property(String name) {
+ return props.get(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridHadoopJob createJob(GridHadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException {
+ try {
+ Class<?> jobCls0 = jobCls;
+
+ if (jobCls0 == null) { // It is enough to have only one class loader with only Hadoop classes.
+ synchronized (GridHadoopDefaultJobInfo.class) {
+ if ((jobCls0 = jobCls) == null) {
+ GridHadoopClassLoader ldr = new GridHadoopClassLoader(null);
+
+ jobCls = jobCls0 = ldr.loadClass(GridHadoopV2Job.class.getName());
+ }
+ }
+ }
+
+ Constructor<?> constructor = jobCls0.getConstructor(GridHadoopJobId.class, GridHadoopDefaultJobInfo.class,
+ IgniteLogger.class);
+
+ return (GridHadoopJob)constructor.newInstance(jobId, this, log);
+ }
+ catch (Exception e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasCombiner() {
+ return hasCombiner;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasReducer() {
+ return reducers() > 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int reducers() {
+ return numReduces;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String jobName() {
+ return jobName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String user() {
+ return user;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeString(out, jobName);
+ U.writeString(out, user);
+
+ out.writeBoolean(hasCombiner);
+ out.writeInt(numReduces);
+
+ U.writeStringMap(out, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobName = U.readString(in);
+ user = U.readString(in);
+
+ hasCombiner = in.readBoolean();
+ numReduces = in.readInt();
+
+ props = U.readStringMap(in);
+ }
+
+ /**
+ * @return Properties of the job.
+ */
+ public Map<String, String> properties() {
+ return props;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopImpl.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopImpl.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopImpl.java
index 79f099d..55e3690 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopImpl.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopImpl.java
@@ -20,7 +20,6 @@ package org.apache.ignite.internal.processors.hadoop;
import org.apache.ignite.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.util.*;
-import org.apache.ignite.hadoop.*;
import org.jetbrains.annotations.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSetup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSetup.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSetup.java
new file mode 100644
index 0000000..41fed96
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopSetup.java
@@ -0,0 +1,506 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.net.*;
+import java.nio.file.*;
+import java.text.*;
+import java.util.*;
+
+import static org.apache.ignite.internal.GridProductImpl.*;
+
+/**
+ * Setup tool to configure Hadoop client.
+ */
+public class GridHadoopSetup {
+ /** */
+ public static final String WINUTILS_EXE = "winutils.exe";
+
+ /** */
+ private static final FilenameFilter IGNITE_JARS = new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return name.startsWith("ignite-") && name.endsWith(".jar");
+ }
+ };
+
+ /**
+ * The main method.
+ * @param ignore Params.
+ * @throws IOException If fails.
+ */
+ public static void main(String[] ignore) throws IOException {
+ X.println(
+ " __________ ________________ ",
+ " / _/ ___/ |/ / _/_ __/ __/ ",
+ " _/ // (_ / // / / / / _/ ",
+ "/___/\\___/_/|_/___/ /_/ /___/ ",
+ " for Apache Hadoop ",
+ " ");
+
+ println("Version " + ACK_VER);
+
+ configureHadoop();
+ }
+
+ /**
+ * This operation prepares the clean unpacked Hadoop distributive to work as client with GridGain-Hadoop.
+ * It performs these operations:
+ * <ul>
+ * <li>Check for setting of HADOOP_HOME environment variable.</li>
+ * <li>Try to resolve HADOOP_COMMON_HOME or evaluate it relative to HADOOP_HOME.</li>
+ * <li>In Windows check if winutils.exe exists and try to fix issue with some restrictions.</li>
+ * <li>In Windows check new line character issues in CMD scripts.</li>
+ * <li>Scan Hadoop lib directory to detect GridGain JARs. If these don't exist tries to create ones.</li>
+ * </ul>
+ */
+ private static void configureHadoop() {
+ String gridgainHome = U.getGridGainHome();
+
+ println("IGNITE_HOME is set to '" + gridgainHome + "'.");
+
+ checkGridGainHome(gridgainHome);
+
+ String homeVar = "HADOOP_HOME";
+ String hadoopHome = System.getenv(homeVar);
+
+ if (F.isEmpty(hadoopHome)) {
+ homeVar = "HADOOP_PREFIX";
+ hadoopHome = System.getenv(homeVar);
+ }
+
+ if (F.isEmpty(hadoopHome))
+ exit("Neither HADOOP_HOME nor HADOOP_PREFIX environment variable is set. Please set one of them to a " +
+ "valid Hadoop installation directory and run setup tool again.", null);
+
+ hadoopHome = hadoopHome.replaceAll("\"", "");
+
+ println(homeVar + " is set to '" + hadoopHome + "'.");
+
+ String hiveHome = System.getenv("HIVE_HOME");
+
+ if (!F.isEmpty(hiveHome)) {
+ hiveHome = hiveHome.replaceAll("\"", "");
+
+ println("HIVE_HOME is set to '" + hiveHome + "'.");
+ }
+
+ File hadoopDir = new File(hadoopHome);
+
+ if (!hadoopDir.exists())
+ exit("Hadoop installation folder does not exist.", null);
+
+ if (!hadoopDir.isDirectory())
+ exit("HADOOP_HOME must point to a directory.", null);
+
+ if (!hadoopDir.canRead())
+ exit("Hadoop installation folder can not be read. Please check permissions.", null);
+
+ File hadoopCommonDir;
+
+ String hadoopCommonHome = System.getenv("HADOOP_COMMON_HOME");
+
+ if (F.isEmpty(hadoopCommonHome)) {
+ hadoopCommonDir = new File(hadoopDir, "share/hadoop/common");
+
+ println("HADOOP_COMMON_HOME is not set, will use '" + hadoopCommonDir.getPath() + "'.");
+ }
+ else {
+ println("HADOOP_COMMON_HOME is set to '" + hadoopCommonHome + "'.");
+
+ hadoopCommonDir = new File(hadoopCommonHome);
+ }
+
+ if (!hadoopCommonDir.canRead())
+ exit("Failed to read Hadoop common dir in '" + hadoopCommonHome + "'.", null);
+
+ File hadoopCommonLibDir = new File(hadoopCommonDir, "lib");
+
+ if (!hadoopCommonLibDir.canRead())
+ exit("Failed to read Hadoop 'lib' folder in '" + hadoopCommonLibDir.getPath() + "'.", null);
+
+ if (U.isWindows()) {
+ checkJavaPathSpaces();
+
+ File hadoopBinDir = new File(hadoopDir, "bin");
+
+ if (!hadoopBinDir.canRead())
+ exit("Failed to read subdirectory 'bin' in HADOOP_HOME.", null);
+
+ File winutilsFile = new File(hadoopBinDir, WINUTILS_EXE);
+
+ if (!winutilsFile.exists()) {
+ if (ask("File '" + WINUTILS_EXE + "' does not exist. " +
+ "It may be replaced by a stub. Create it?")) {
+ println("Creating file stub '" + winutilsFile.getAbsolutePath() + "'.");
+
+ boolean ok = false;
+
+ try {
+ ok = winutilsFile.createNewFile();
+ }
+ catch (IOException ignore) {
+ // No-op.
+ }
+
+ if (!ok)
+ exit("Failed to create '" + WINUTILS_EXE + "' file. Please check permissions.", null);
+ }
+ else
+ println("Ok. But Hadoop client probably will not work on Windows this way...");
+ }
+
+ processCmdFiles(hadoopDir, "bin", "sbin", "libexec");
+ }
+
+ File gridgainLibs = new File(new File(gridgainHome), "libs");
+
+ if (!gridgainLibs.exists())
+ exit("GridGain 'libs' folder is not found.", null);
+
+ Collection<File> jarFiles = new ArrayList<>();
+
+ addJarsInFolder(jarFiles, gridgainLibs);
+ addJarsInFolder(jarFiles, new File(gridgainLibs, "gridgain-hadoop"));
+
+ boolean jarsLinksCorrect = true;
+
+ for (File file : jarFiles) {
+ File link = new File(hadoopCommonLibDir, file.getName());
+
+ jarsLinksCorrect &= isJarLinkCorrect(link, file);
+
+ if (!jarsLinksCorrect)
+ break;
+ }
+
+ if (!jarsLinksCorrect) {
+ if (ask("GridGain JAR files are not found in Hadoop 'lib' directory. " +
+ "Create appropriate symbolic links?")) {
+ File[] oldGridGainJarFiles = hadoopCommonLibDir.listFiles(IGNITE_JARS);
+
+ if (oldGridGainJarFiles.length > 0 && ask("The Hadoop 'lib' directory contains JARs from other GridGain " +
+ "installation. They must be deleted to continue. Continue?")) {
+ for (File file : oldGridGainJarFiles) {
+ println("Deleting file '" + file.getAbsolutePath() + "'.");
+
+ if (!file.delete())
+ exit("Failed to delete file '" + file.getPath() + "'.", null);
+ }
+ }
+
+ for (File file : jarFiles) {
+ File targetFile = new File(hadoopCommonLibDir, file.getName());
+
+ try {
+ println("Creating symbolic link '" + targetFile.getAbsolutePath() + "'.");
+
+ Files.createSymbolicLink(targetFile.toPath(), file.toPath());
+ }
+ catch (IOException e) {
+ if (U.isWindows()) {
+ warn("Ability to create symbolic links is required!");
+ warn("On Windows platform you have to grant permission 'Create symbolic links'");
+ warn("to your user or run the Accelerator as Administrator.");
+ }
+
+ exit("Creating symbolic link failed! Check permissions.", e);
+ }
+ }
+ }
+ else
+ println("Ok. But Hadoop client will not be able to talk to GridGain cluster without those JARs in classpath...");
+ }
+
+ File hadoopEtc = new File(hadoopDir, "etc" + File.separator + "hadoop");
+
+ File gridgainDocs = new File(gridgainHome, "docs");
+
+ if (!gridgainDocs.canRead())
+ exit("Failed to read GridGain 'docs' folder at '" + gridgainDocs.getAbsolutePath() + "'.", null);
+
+ if (hadoopEtc.canWrite()) { // TODO Bigtop
+ if (ask("Replace 'core-site.xml' and 'mapred-site.xml' files with preconfigured templates " +
+ "(existing files will be backed up)?")) {
+ replaceWithBackup(new File(gridgainDocs, "core-site.gridgain.xml"), new File(hadoopEtc, "core-site.xml"));
+
+ replaceWithBackup(new File(gridgainDocs, "mapred-site.gridgain.xml"), new File(hadoopEtc, "mapred-site.xml"));
+ }
+ else
+ println("Ok. You can configure them later, the templates are available at GridGain's 'docs' directory...");
+ }
+
+ if (!F.isEmpty(hiveHome)) {
+ File hiveConfDir = new File(hiveHome + File.separator + "conf");
+
+ if (!hiveConfDir.canWrite())
+ warn("Can not write to '" + hiveConfDir.getAbsolutePath() + "'. To run Hive queries you have to " +
+ "configure 'hive-site.xml' manually. The template is available at GridGain's 'docs' directory.");
+ else if (ask("Replace 'hive-site.xml' with preconfigured template (existing file will be backed up)?"))
+ replaceWithBackup(new File(gridgainDocs, "hive-site.gridgain.xml"), new File(hiveConfDir, "hive-site.xml"));
+ else
+ println("Ok. You can configure it later, the template is available at GridGain's 'docs' directory...");
+ }
+
+ println("Apache Hadoop setup is complete.");
+ }
+
+ /**
+ * @param jarFiles Jars.
+ * @param folder Folder.
+ */
+ private static void addJarsInFolder(Collection<File> jarFiles, File folder) {
+ if (!folder.exists())
+ exit("Folder '" + folder.getAbsolutePath() + "' is not found.", null);
+
+ jarFiles.addAll(Arrays.asList(folder.listFiles(IGNITE_JARS)));
+ }
+
+ /**
+ * Checks that JAVA_HOME does not contain space characters.
+ */
+ private static void checkJavaPathSpaces() {
+ String javaHome = System.getProperty("java.home");
+
+ if (javaHome.contains(" ")) {
+ warn("Java installation path contains space characters!");
+ warn("Hadoop client will not be able to start using '" + javaHome + "'.");
+ warn("Please install JRE to path which does not contain spaces and point JAVA_HOME to that installation.");
+ }
+ }
+
+ /**
+ * Checks GridGain home.
+ *
+ * @param ggHome GridGain home.
+ */
+ private static void checkGridGainHome(String ggHome) {
+ URL jarUrl = U.class.getProtectionDomain().getCodeSource().getLocation();
+
+ try {
+ Path jar = Paths.get(jarUrl.toURI());
+ Path gg = Paths.get(ggHome);
+
+ if (!jar.startsWith(gg))
+ exit("GridGain JAR files are not under IGNITE_HOME.", null);
+ }
+ catch (Exception e) {
+ exit(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Replaces target file with source file.
+ *
+ * @param from From.
+ * @param to To.
+ */
+ private static void replaceWithBackup(File from, File to) {
+ if (!from.canRead())
+ exit("Failed to read source file '" + from.getAbsolutePath() + "'.", null);
+
+ println("Replacing file '" + to.getAbsolutePath() + "'.");
+
+ try {
+ U.copy(from, renameToBak(to), true);
+ }
+ catch (IOException e) {
+ exit("Failed to replace file '" + to.getAbsolutePath() + "'.", e);
+ }
+ }
+
+ /**
+ * Renames file for backup.
+ *
+ * @param file File.
+ * @return File.
+ */
+ private static File renameToBak(File file) {
+ DateFormat fmt = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
+
+ if (file.exists() && !file.renameTo(new File(file.getAbsolutePath() + "." + fmt.format(new Date()) + ".bak")))
+ exit("Failed to rename file '" + file.getPath() + "'.", null);
+
+ return file;
+ }
+
+ /**
+ * Checks if link is correct.
+ *
+ * @param link Symbolic link.
+ * @param correctTarget Correct link target.
+ * @return {@code true} If link target is correct.
+ */
+ private static boolean isJarLinkCorrect(File link, File correctTarget) {
+ if (!Files.isSymbolicLink(link.toPath()))
+ return false; // It is a real file or it does not exist.
+
+ Path target = null;
+
+ try {
+ target = Files.readSymbolicLink(link.toPath());
+ }
+ catch (IOException e) {
+ exit("Failed to read symbolic link: " + link.getAbsolutePath(), e);
+ }
+
+ return Files.exists(target) && target.toFile().equals(correctTarget);
+ }
+
+ /**
+ * Writes the question end read the boolean answer from the console.
+ *
+ * @param question Question to write.
+ * @return {@code true} if user inputs 'Y' or 'y', {@code false} otherwise.
+ */
+ private static boolean ask(String question) {
+ X.println();
+ X.print(" < " + question + " (Y/N): ");
+
+ String answer = null;
+
+ if (!F.isEmpty(System.getenv("IGNITE_HADOOP_SETUP_YES")))
+ answer = "Y";
+ else {
+ BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+
+ try {
+ answer = br.readLine();
+ }
+ catch (IOException e) {
+ exit("Failed to read answer: " + e.getMessage(), e);
+ }
+ }
+
+ if (answer != null && "Y".equals(answer.toUpperCase().trim())) {
+ X.println(" > Yes.");
+
+ return true;
+ }
+ else {
+ X.println(" > No.");
+
+ return false;
+ }
+ }
+
+ /**
+ * Exit with message.
+ *
+ * @param msg Exit message.
+ */
+ private static void exit(String msg, Exception e) {
+ X.println(" ");
+ X.println(" # " + msg);
+ X.println(" # Setup failed, exiting... ");
+
+ if (e != null && !F.isEmpty(System.getenv("IGNITE_HADOOP_SETUP_DEBUG")))
+ e.printStackTrace();
+
+ System.exit(1);
+ }
+
+ /**
+ * Prints message.
+ *
+ * @param msg Message.
+ */
+ private static void println(String msg) {
+ X.println(" > " + msg);
+ }
+
+ /**
+ * Prints warning.
+ *
+ * @param msg Message.
+ */
+ private static void warn(String msg) {
+ X.println(" ! " + msg);
+ }
+
+ /**
+ * Checks that CMD files have valid MS Windows new line characters. If not, writes question to console and reads the
+ * answer. If it's 'Y' then backups original files and corrects invalid new line characters.
+ *
+ * @param rootDir Root directory to process.
+ * @param dirs Directories inside of the root to process.
+ */
+ private static void processCmdFiles(File rootDir, String... dirs) {
+ boolean answer = false;
+
+ for (String dir : dirs) {
+ File subDir = new File(rootDir, dir);
+
+ File[] cmdFiles = subDir.listFiles(new FilenameFilter() {
+ @Override public boolean accept(File dir, String name) {
+ return name.toLowerCase().endsWith(".cmd");
+ }
+ });
+
+ for (File file : cmdFiles) {
+ String content = null;
+
+ try (Scanner scanner = new Scanner(file)) {
+ content = scanner.useDelimiter("\\Z").next();
+ }
+ catch (FileNotFoundException e) {
+ exit("Failed to read file '" + file + "'.", e);
+ }
+
+ boolean invalid = false;
+
+ for (int i = 0; i < content.length(); i++) {
+ if (content.charAt(i) == '\n' && (i == 0 || content.charAt(i - 1) != '\r')) {
+ invalid = true;
+
+ break;
+ }
+ }
+
+ if (invalid) {
+ answer = answer || ask("One or more *.CMD files has invalid new line character. Replace them?");
+
+ if (!answer) {
+ println("Ok. But Windows most probably will fail to execute them...");
+
+ return;
+ }
+
+ println("Fixing newline characters in file '" + file.getAbsolutePath() + "'.");
+
+ renameToBak(file);
+
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) {
+ for (int i = 0; i < content.length(); i++) {
+ if (content.charAt(i) == '\n' && (i == 0 || content.charAt(i - 1) != '\r'))
+ writer.write("\r");
+
+ writer.write(content.charAt(i));
+ }
+ }
+ catch (IOException e) {
+ exit("Failed to write file '" + file.getPath() + "': " + e.getMessage(), e);
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopUtils.java
index cfd9786..b6ff150 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopUtils.java
@@ -20,10 +20,12 @@ package org.apache.ignite.internal.processors.hadoop;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobPriority;
+import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.*;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.v2.*;
import org.apache.ignite.internal.util.typedef.internal.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessor.java
index c0c8e23..4ef9e35 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessor.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgniteHadoopProcessor.java
@@ -19,7 +19,6 @@ package org.apache.ignite.internal.processors.hadoop;
import org.apache.ignite.*;
import org.apache.ignite.internal.*;
-import org.apache.ignite.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.jobtracker.*;
import org.apache.ignite.internal.processors.hadoop.planner.*;
import org.apache.ignite.internal.processors.hadoop.shuffle.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCounterAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCounterAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCounterAdapter.java
index ec259b0..9e46846 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCounterAdapter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCounterAdapter.java
@@ -17,7 +17,7 @@
package org.apache.ignite.internal.processors.hadoop.counter;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.jetbrains.annotations.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCountersImpl.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCountersImpl.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCountersImpl.java
index 5dcab2f..92d54af 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCountersImpl.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopCountersImpl.java
@@ -18,7 +18,7 @@
package org.apache.ignite.internal.processors.hadoop.counter;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.lang.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.jdk8.backport.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopFSCounterWriter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopFSCounterWriter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopFSCounterWriter.java
index 171ba62..5e9f9af 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopFSCounterWriter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopFSCounterWriter.java
@@ -18,9 +18,10 @@
package org.apache.ignite.internal.processors.hadoop.counter;
import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.*;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.typedef.*;
import java.io.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopLongCounter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopLongCounter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopLongCounter.java
index 67960b6..67af49f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopLongCounter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopLongCounter.java
@@ -17,7 +17,8 @@
package org.apache.ignite.internal.processors.hadoop.counter;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
+
import java.io.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopPerformanceCounter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopPerformanceCounter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopPerformanceCounter.java
index e790124..d5ceebf 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopPerformanceCounter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/counter/GridHadoopPerformanceCounter.java
@@ -17,7 +17,7 @@
package org.apache.ignite.internal.processors.hadoop.counter;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.jetbrains.annotations.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobMetadata.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobMetadata.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobMetadata.java
index c83a5b9..b124312 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobMetadata.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobMetadata.java
@@ -17,7 +17,7 @@
package org.apache.ignite.internal.processors.hadoop.jobtracker;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.counter.*;
import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
import org.apache.ignite.internal.util.tostring.*;
@@ -26,7 +26,7 @@ import org.apache.ignite.internal.util.typedef.internal.*;
import java.io.*;
import java.util.*;
-import static org.apache.ignite.hadoop.GridHadoopJobPhase.*;
+import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobPhase.*;
/**
* Hadoop job metadata. Internal object used for distributed job state tracking.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobTracker.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobTracker.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobTracker.java
index 6e35f47..a2e5e40 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobTracker.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/GridHadoopJobTracker.java
@@ -22,18 +22,17 @@ import org.apache.ignite.cache.*;
import org.apache.ignite.cache.query.*;
import org.apache.ignite.events.*;
import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.cache.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.hadoop.*;
import org.apache.ignite.internal.managers.eventstorage.*;
+import org.apache.ignite.internal.processors.cache.*;
import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.counter.*;
import org.apache.ignite.internal.processors.hadoop.taskexecutor.*;
import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*;
+import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.future.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
import org.jdk8.backport.*;
import org.jetbrains.annotations.*;
@@ -45,8 +44,8 @@ import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import static java.util.concurrent.TimeUnit.*;
-import static org.apache.ignite.hadoop.GridHadoopJobPhase.*;
-import static org.apache.ignite.hadoop.GridHadoopTaskType.*;
+import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobPhase.*;
+import static org.apache.ignite.internal.processors.hadoop.GridHadoopTaskType.*;
import static org.apache.ignite.internal.processors.hadoop.taskexecutor.GridHadoopTaskState.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlan.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlan.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlan.java
index 84fb34a..7988403 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlan.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlan.java
@@ -17,7 +17,7 @@
package org.apache.ignite.internal.processors.hadoop.planner;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.jetbrains.annotations.*;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlanner.java
index d98d8f4..df868fb 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/GridHadoopDefaultMapReducePlanner.java
@@ -21,12 +21,12 @@ import org.apache.ignite.*;
import org.apache.ignite.cluster.*;
import org.apache.ignite.fs.*;
import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.fs.*;
-import org.apache.ignite.resources.*;
-import org.apache.ignite.hadoop.*;
import org.apache.ignite.internal.fs.hadoop.*;
+import org.apache.ignite.internal.processors.fs.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.resources.*;
import org.jetbrains.annotations.*;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobCountersTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobCountersTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobCountersTask.java
index c4f5f9b..37073d9 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobCountersTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobCountersTask.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.processors.hadoop.proto;
import org.apache.ignite.*;
import org.apache.ignite.compute.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobStatusTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobStatusTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobStatusTask.java
index 19adf3b..de4f89c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobStatusTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolJobStatusTask.java
@@ -20,9 +20,9 @@ package org.apache.ignite.internal.processors.hadoop.proto;
import org.apache.ignite.*;
import org.apache.ignite.compute.*;
import org.apache.ignite.internal.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.lang.*;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolKillJobTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolKillJobTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolKillJobTask.java
index cde85d8..384bc23 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolKillJobTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolKillJobTask.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.processors.hadoop.proto;
import org.apache.ignite.*;
import org.apache.ignite.compute.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolNextTaskIdTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolNextTaskIdTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolNextTaskIdTask.java
index d441e65..a1968b4 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolNextTaskIdTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolNextTaskIdTask.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.processors.hadoop.proto;
import org.apache.ignite.*;
import org.apache.ignite.compute.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
/**
* Task to get the next job ID.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolSubmitJobTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolSubmitJobTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolSubmitJobTask.java
index 5411a87..c734acd 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolSubmitJobTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolSubmitJobTask.java
@@ -19,11 +19,11 @@ package org.apache.ignite.internal.processors.hadoop.proto;
import org.apache.ignite.*;
import org.apache.ignite.compute.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import java.util.*;
-import static org.apache.ignite.hadoop.GridHadoopJobPhase.*;
+import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobPhase.*;
/**
* Submit job task.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskAdapter.java
index 01a68e1..081a679 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskAdapter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/proto/GridHadoopProtocolTaskAdapter.java
@@ -21,9 +21,9 @@ import org.apache.ignite.*;
import org.apache.ignite.cluster.*;
import org.apache.ignite.compute.*;
import org.apache.ignite.internal.*;
-import org.apache.ignite.resources.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.resources.*;
import org.jetbrains.annotations.*;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffle.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffle.java
index c186953..396124e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffle.java
@@ -20,8 +20,6 @@ package org.apache.ignite.internal.processors.hadoop.shuffle;
import org.apache.ignite.*;
import org.apache.ignite.cluster.*;
import org.apache.ignite.internal.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.message.*;
import org.apache.ignite.internal.util.future.*;
@@ -29,6 +27,7 @@ import org.apache.ignite.internal.util.lang.*;
import org.apache.ignite.internal.util.offheap.unsafe.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.lang.*;
import java.util.*;
import java.util.concurrent.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleAck.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleAck.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleAck.java
index a16ef23..a8a52a9 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleAck.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleAck.java
@@ -17,7 +17,7 @@
package org.apache.ignite.internal.processors.hadoop.shuffle;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.message.*;
import org.apache.ignite.internal.util.tostring.*;
import org.apache.ignite.internal.util.typedef.internal.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleJob.java
index 54ef6c7..491205f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleJob.java
@@ -19,9 +19,7 @@ package org.apache.ignite.internal.processors.hadoop.shuffle;
import org.apache.ignite.*;
import org.apache.ignite.internal.*;
-import org.apache.ignite.lang.*;
-import org.apache.ignite.thread.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.counter.*;
import org.apache.ignite.internal.processors.hadoop.shuffle.collections.*;
import org.apache.ignite.internal.util.future.*;
@@ -31,12 +29,14 @@ import org.apache.ignite.internal.util.offheap.unsafe.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.apache.ignite.internal.util.worker.*;
+import org.apache.ignite.lang.*;
+import org.apache.ignite.thread.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
-import static org.apache.ignite.hadoop.GridHadoopJobProperty.*;
+import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobProperty.*;
import static org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleMessage.java
index 1410c35..f68ec7e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleMessage.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/GridHadoopShuffleMessage.java
@@ -18,7 +18,7 @@
package org.apache.ignite.internal.processors.hadoop.shuffle;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.message.*;
import org.apache.ignite.internal.util.tostring.*;
import org.apache.ignite.internal.util.typedef.internal.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimap.java
index 435bdd0..32db722 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimap.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopConcurrentHashMultimap.java
@@ -18,7 +18,7 @@
package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.offheap.unsafe.*;
import org.apache.ignite.internal.util.typedef.internal.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimap.java
index ebf5ab6..2795b77 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimap.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimap.java
@@ -18,7 +18,7 @@
package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.offheap.unsafe.*;
import org.apache.ignite.internal.util.typedef.internal.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimapBase.java
index 07fe856..f7e1362 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimapBase.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopHashMultimapBase.java
@@ -18,7 +18,7 @@
package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.offheap.unsafe.*;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimap.java
index 267b16c..b8eb12c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimap.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimap.java
@@ -18,7 +18,7 @@
package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.jetbrains.annotations.*;
import java.io.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimapBase.java
index 68243d7..2d8660f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimapBase.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopMultimapBase.java
@@ -18,9 +18,9 @@
package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
-import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.hadoop.shuffle.streams.*;
+import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.offheap.unsafe.*;
import org.jetbrains.annotations.*;
@@ -28,7 +28,7 @@ import java.io.*;
import java.util.*;
import java.util.concurrent.*;
-import static org.apache.ignite.hadoop.GridHadoopJobProperty.*;
+import static org.apache.ignite.internal.processors.hadoop.GridHadoopJobProperty.*;
/**
* Base class for all multimaps.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/36b439d9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipList.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipList.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipList.java
index 13582f6..a2c626c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipList.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/GridHadoopSkipList.java
@@ -18,7 +18,7 @@
package org.apache.ignite.internal.processors.hadoop.shuffle.collections;
import org.apache.ignite.*;
-import org.apache.ignite.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.offheap.unsafe.*;
import org.apache.ignite.internal.util.typedef.internal.*;