You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2018/06/18 21:09:39 UTC
[09/10] asterixdb git commit: [NO ISSUE] Remove obsolete support for
older HDFS versions
[NO ISSUE] Remove obsolete support for older HDFS versions
Change-Id: I84f1bced874f4f262324fa6114a2394c6c5bb3c5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2697
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2d90c0c1
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2d90c0c1
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2d90c0c1
Branch: refs/heads/master
Commit: 2d90c0c1f20fbd282bdad1c99bcfe1b282f86ecf
Parents: 60ce519
Author: Michael Blow <mb...@apache.org>
Authored: Sat Jun 16 08:59:54 2018 -0400
Committer: Michael Blow <mb...@apache.org>
Committed: Mon Jun 18 10:49:12 2018 -0700
----------------------------------------------------------------------
asterixdb/asterix-coverage/pom.xml | 1 -
asterixdb/asterix-external-data/pom.xml | 2 +-
asterixdb/pom.xml | 4 +-
.../hyracks-hdfs/hyracks-hdfs-1.x/pom.xml | 116 ------
.../org/apache/hyracks/hdfs/ContextFactory.java | 56 ---
.../hyracks/hdfs/MiniDFSClusterFactory.java | 35 --
.../hyracks-hdfs/hyracks-hdfs-2.x/pom.xml | 232 -----------
.../org/apache/hyracks/hdfs/ContextFactory.java | 57 ---
.../hyracks/hdfs/MiniDFSClusterFactory.java | 38 --
.../hyracks-hdfs/hyracks-hdfs-core/pom.xml | 409 -------------------
.../hyracks/hdfs/api/IKeyValueParser.java | 62 ---
.../hdfs/api/IKeyValueParserFactory.java | 46 ---
.../apache/hyracks/hdfs/api/INcCollection.java | 29 --
.../hyracks/hdfs/api/INcCollectionBuilder.java | 35 --
.../apache/hyracks/hdfs/api/ITupleWriter.java | 61 ---
.../hyracks/hdfs/api/ITupleWriterFactory.java | 40 --
.../hyracks/hdfs/dataflow/ConfFactory.java | 57 ---
.../dataflow/HDFSReadOperatorDescriptor.java | 153 -------
.../dataflow/HDFSWriteOperatorDescriptor.java | 134 ------
.../hdfs/dataflow/InputSplitsFactory.java | 109 -----
.../hdfs/lib/RawBinaryComparatorFactory.java | 48 ---
.../hdfs/lib/RawBinaryHashFunctionFactory.java | 47 ---
.../hdfs/lib/TextKeyValueParserFactory.java | 69 ----
.../hdfs/lib/TextTupleWriterFactory.java | 64 ---
.../IPProximityNcCollectionBuilder.java | 147 -------
.../scheduler/RackAwareNcCollectionBuilder.java | 215 ----------
.../hyracks/hdfs/scheduler/Scheduler.java | 405 ------------------
.../hyracks/hdfs2/dataflow/ConfFactory.java | 57 ---
.../hdfs2/dataflow/FileSplitsFactory.java | 109 -----
.../dataflow/HDFSReadOperatorDescriptor.java | 167 --------
.../dataflow/HDFSWriteOperatorDescriptor.java | 133 ------
.../hyracks/hdfs2/scheduler/Scheduler.java | 104 -----
.../hdfs2/scheduler/WrappedFileSplit.java | 69 ----
.../hyracks/hdfs/dataflow/DataflowTest.java | 217 ----------
.../hyracks/hdfs/scheduler/SchedulerTest.java | 268 ------------
.../apache/hyracks/hdfs/utils/HyracksUtils.java | 99 -----
.../hyracks/hdfs2/dataflow/DataflowTest.java | 134 ------
.../hyracks/hdfs2/scheduler/SchedulerTest.java | 170 --------
.../src/test/resources/data/customer.tbl | 150 -------
.../src/test/resources/expected/part-0 | 150 -------
.../test/resources/hadoop/conf/core-site.xml | 35 --
.../test/resources/hadoop/conf/hdfs-site.xml | 41 --
.../test/resources/hadoop/conf/log4j.properties | 112 -----
.../test/resources/hadoop/conf/mapred-site.xml | 43 --
.../src/test/resources/topology.xml | 51 ---
hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml | 158 ++++++-
.../org/apache/hyracks/hdfs/ContextFactory.java | 57 +++
.../hyracks/hdfs/api/IKeyValueParser.java | 62 +++
.../hdfs/api/IKeyValueParserFactory.java | 46 +++
.../apache/hyracks/hdfs/api/INcCollection.java | 29 ++
.../hyracks/hdfs/api/INcCollectionBuilder.java | 35 ++
.../apache/hyracks/hdfs/api/ITupleWriter.java | 61 +++
.../hyracks/hdfs/api/ITupleWriterFactory.java | 40 ++
.../hyracks/hdfs/dataflow/ConfFactory.java | 57 +++
.../dataflow/HDFSReadOperatorDescriptor.java | 153 +++++++
.../dataflow/HDFSWriteOperatorDescriptor.java | 134 ++++++
.../hdfs/dataflow/InputSplitsFactory.java | 109 +++++
.../hdfs/lib/RawBinaryComparatorFactory.java | 48 +++
.../hdfs/lib/RawBinaryHashFunctionFactory.java | 47 +++
.../hdfs/lib/TextKeyValueParserFactory.java | 69 ++++
.../hdfs/lib/TextTupleWriterFactory.java | 64 +++
.../IPProximityNcCollectionBuilder.java | 147 +++++++
.../scheduler/RackAwareNcCollectionBuilder.java | 215 ++++++++++
.../hyracks/hdfs/scheduler/Scheduler.java | 405 ++++++++++++++++++
.../hyracks/hdfs2/dataflow/ConfFactory.java | 57 +++
.../hdfs2/dataflow/FileSplitsFactory.java | 109 +++++
.../dataflow/HDFSReadOperatorDescriptor.java | 167 ++++++++
.../dataflow/HDFSWriteOperatorDescriptor.java | 133 ++++++
.../hyracks/hdfs2/scheduler/Scheduler.java | 104 +++++
.../hdfs2/scheduler/WrappedFileSplit.java | 69 ++++
.../hyracks/hdfs/MiniDFSClusterFactory.java | 38 ++
.../hyracks/hdfs/dataflow/DataflowTest.java | 217 ++++++++++
.../hyracks/hdfs/scheduler/SchedulerTest.java | 268 ++++++++++++
.../apache/hyracks/hdfs/utils/HyracksUtils.java | 99 +++++
.../hyracks/hdfs2/dataflow/DataflowTest.java | 134 ++++++
.../hyracks/hdfs2/scheduler/SchedulerTest.java | 170 ++++++++
.../src/test/resources/data/customer.tbl | 150 +++++++
.../src/test/resources/expected/part-0 | 150 +++++++
.../test/resources/hadoop/conf/core-site.xml | 35 ++
.../test/resources/hadoop/conf/hdfs-site.xml | 41 ++
.../test/resources/hadoop/conf/log4j.properties | 112 +++++
.../test/resources/hadoop/conf/mapred-site.xml | 43 ++
.../src/test/resources/topology.xml | 51 +++
83 files changed, 4080 insertions(+), 4783 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/asterixdb/asterix-coverage/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-coverage/pom.xml b/asterixdb/asterix-coverage/pom.xml
index 1f578b2..65be538 100644
--- a/asterixdb/asterix-coverage/pom.xml
+++ b/asterixdb/asterix-coverage/pom.xml
@@ -83,7 +83,6 @@
<classfiles>
<fileset dir="${asterix.dir}">
<include name="**/target/classes/**/*.class" />
- <exclude name="**/hyracks-hdfs/hyracks-hdfs-1.x/**" />
</fileset>
</classfiles>
<sourcefiles encoding="UTF-8">
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/asterixdb/asterix-external-data/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index 78549b3..964f7f9 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -221,7 +221,7 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-core</artifactId>
+ <artifactId>hyracks-hdfs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.asterix</groupId>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/asterixdb/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 11a8cba..eaee996 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -1113,7 +1113,7 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-core</artifactId>
+ <artifactId>hyracks-hdfs</artifactId>
<version>${hyracks.version}</version>
</dependency>
<dependency>
@@ -1129,7 +1129,7 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
+ <artifactId>hyracks-hdfs</artifactId>
<version>${hyracks.version}</version>
<type>test-jar</type>
</dependency>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/pom.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/pom.xml
deleted file mode 100644
index aa03551..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/pom.xml
+++ /dev/null
@@ -1,116 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>hyracks-hdfs-1.x</artifactId>
- <name>hyracks-hdfs-1.x</name>
- <parent>
- <artifactId>hyracks-hdfs</artifactId>
- <groupId>org.apache.hyracks</groupId>
- <version>0.3.4-SNAPSHOT</version>
- </parent>
- <properties>
- <root.dir>${basedir}/../../..</root.dir>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <id>hadoop-0.20.2</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>0.20.2</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-test</artifactId>
- <version>0.20.2</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>1.0.4</value>
- </property>
- </activation>
- <id>hadoop-1.0.4</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>1.0.4</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>1.0.4</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-test</artifactId>
- <version>1.0.4</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-api</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java
deleted file mode 100644
index ecee076..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.hdfs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-/**
- * The wrapper to generate TaskTattemptContext
- */
-public class ContextFactory {
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- public TaskAttemptContext createContext(Configuration conf, TaskAttemptID tid) throws HyracksDataException {
- try {
- return new Mapper().new Context(conf, tid, null, null, null, null, null);
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- public TaskAttemptContext createContext(Configuration conf, int partition) throws HyracksDataException {
- try {
- TaskAttemptID tid = new TaskAttemptID("", 0, true, partition, 0);
- return new TaskAttemptContext(conf, tid);
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- public JobContext createJobContext(Configuration conf) {
- return new JobContext(conf, new JobID("0", 0));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/test/java/org/apache/hyracks/hdfs/MiniDFSClusterFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/test/java/org/apache/hyracks/hdfs/MiniDFSClusterFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/test/java/org/apache/hyracks/hdfs/MiniDFSClusterFactory.java
deleted file mode 100644
index d17f1bf..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/test/java/org/apache/hyracks/hdfs/MiniDFSClusterFactory.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.hdfs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class MiniDFSClusterFactory {
-
- public MiniDFSCluster getMiniDFSCluster(Configuration conf, int numberOfNC) throws HyracksDataException {
- try {
- return new MiniDFSCluster(conf, numberOfNC, true, null);
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/pom.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/pom.xml
deleted file mode 100644
index 2160294..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/pom.xml
+++ /dev/null
@@ -1,232 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <name>hyracks-hdfs-2.x</name>
- <parent>
- <artifactId>hyracks-hdfs</artifactId>
- <groupId>org.apache.hyracks</groupId>
- <version>0.3.4-SNAPSHOT</version>
- </parent>
- <properties>
- <root.dir>${basedir}/../../..</root.dir>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <activation>
- <activeByDefault>true</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>2.2.0</value>
- </property>
- </activation>
- <id>hadoop-2.2.0</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <activation>
- <activeByDefault>true</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>0.23.1</value>
- </property>
- </activation>
- <id>hadoop-0.23.1</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <id>hadoop-0.23.6</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>0.23.6</value>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>0.23.6</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>0.23.6</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>0.23.6</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>0.23.6</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>cdh-4.2</value>
- </property>
- </activation>
- <id>cdh-4.2</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>2.0.0-cdh4.2.0</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>2.0.0-cdh4.2.0</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>2.0.0-cdh4.2.0</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>2.0.0-cdh4.2.0</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>cdh-4.1</value>
- </property>
- </activation>
- <id>cdh-4.1</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>2.0.0-cdh4.1.0</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>2.0.0-cdh4.1.0</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>2.0.0-cdh4.1.0</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>2.0.0-cdh4.1.0</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-api</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java
deleted file mode 100644
index 96b0a23..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.hdfs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.task.JobContextImpl;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-/**
- * The wrapper to generate TaskTattemptContext
- */
-public class ContextFactory {
-
- public TaskAttemptContext createContext(Configuration conf, TaskAttemptID tid) throws HyracksDataException {
- try {
- return new TaskAttemptContextImpl(conf, tid);
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- public TaskAttemptContext createContext(Configuration conf, int partition) throws HyracksDataException {
- try {
- TaskAttemptID tid = new TaskAttemptID("", 0, TaskType.REDUCE, partition, 0);
- return new TaskAttemptContextImpl(conf, tid);
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- public JobContext createJobContext(Configuration conf) {
- return new JobContextImpl(conf, new JobID("0", 0));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/test/java/org/apache/hyracks/hdfs/MiniDFSClusterFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/test/java/org/apache/hyracks/hdfs/MiniDFSClusterFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/test/java/org/apache/hyracks/hdfs/MiniDFSClusterFactory.java
deleted file mode 100644
index 2884e28..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/test/java/org/apache/hyracks/hdfs/MiniDFSClusterFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.hdfs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class MiniDFSClusterFactory {
-
- public MiniDFSCluster getMiniDFSCluster(Configuration conf, int numberOfNC) throws HyracksDataException {
- try {
- MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
- builder.numDataNodes(numberOfNC);
- MiniDFSCluster dfsCluster = builder.build();
- return dfsCluster;
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
deleted file mode 100644
index 99112c0..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
+++ /dev/null
@@ -1,409 +0,0 @@
-<?xml version="1.0"?>
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>hyracks-hdfs-core</artifactId>
- <name>hyracks-hdfs-core</name>
- <parent>
- <artifactId>hyracks-hdfs</artifactId>
- <groupId>org.apache.hyracks</groupId>
- <version>0.3.4-SNAPSHOT</version>
- </parent>
- <properties>
- <root.dir>${basedir}/../../..</root.dir>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <failOnWarning>true</failOnWarning>
- <outputXML>true</outputXML>
- <ignoredUnusedDeclaredDependencies>org.apache.hadoop:hadoop*::</ignoredUnusedDeclaredDependencies>
- </configuration>
- <executions>
- <execution>
- <phase>process-test-classes</phase>
- <goals>
- <goal>analyze-only</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes combine.children="append">
- <exclude>src/test/resources/data/customer.tbl</exclude>
- <exclude>src/test/resources/expected/part-0</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <id>hadoop-0.20.2</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-1.x</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-1.x</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-test</artifactId>
- <version>0.20.2</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>1.0.4</value>
- </property>
- </activation>
- <id>hadoop-1.0.4</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-1.x</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-1.x</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>1.0.4</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-test</artifactId>
- <version>1.0.4</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <activation>
- <activeByDefault>true</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>2.2.0</value>
- </property>
- </activation>
- <id>hadoop-2.2.0</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-control-common</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <type>test-jar</type>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <type>test-jar</type>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>0.23.1</value>
- </property>
- </activation>
- <id>hadoop-0.23.1</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>0.23.6</value>
- </property>
- </activation>
- <id>hadoop-0.23.6</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>0.23.6</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>cdh-4.1</value>
- </property>
- </activation>
- <id>cdh-4.1</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>2.0.0-cdh4.1.0</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>hadoop</name>
- <value>cdh-4.2</value>
- </property>
- </activation>
- <id>cdh-4.2</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-2.x</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>2.0.0-cdh4.2.0</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-api</artifactId>
- <version>${project.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-dataflow-std</artifactId>
- <version>${project.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-dataflow-common</artifactId>
- <version>${project.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-control-cc</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-control-nc</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-test-support</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-util</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/IKeyValueParser.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/IKeyValueParser.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/IKeyValueParser.java
deleted file mode 100644
index 0e2e8df..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/IKeyValueParser.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.hdfs.api;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-/**
- * Users need to implement this interface to use the HDFSReadOperatorDescriptor.
- *
- * @param <K>
- * the key type
- * @param <V>
- * the value type
- */
-public interface IKeyValueParser<K, V> {
-
- /**
- * Initialize the key value parser.
- *
- * @param writer
- * The hyracks writer for outputting data.
- * @throws HyracksDataException
- */
- public void open(IFrameWriter writer) throws HyracksDataException;
-
- /**
- * @param key
- * @param value
- * @param writer
- * @param fileName
- * @throws HyracksDataException
- */
- public void parse(K key, V value, IFrameWriter writer, String fileString) throws HyracksDataException;
-
- /**
- * Flush the residual tuples in the internal buffer to the writer.
- * This method is called in the close() of HDFSReadOperatorDescriptor.
- *
- * @param writer
- * The hyracks writer for outputting data.
- * @throws HyracksDataException
- */
- public void close(IFrameWriter writer) throws HyracksDataException;
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/IKeyValueParserFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/IKeyValueParserFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/IKeyValueParserFactory.java
deleted file mode 100644
index 1cde639..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/IKeyValueParserFactory.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.hdfs.api;
-
-import java.io.Serializable;
-
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-/**
- * Users need to implement this interface to use the HDFSReadOperatorDescriptor.
- *
- * @param <K>
- * the key type
- * @param <V>
- * the value type
- */
-public interface IKeyValueParserFactory<K, V> extends Serializable {
-
- /**
- * This method creates a key-value parser.
- *
- * @param ctx
- * the IHyracksTaskContext
- * @return a key-value parser instance.
- */
- public IKeyValueParser<K, V> createKeyValueParser(IHyracksTaskContext ctx) throws HyracksDataException;
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/INcCollection.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/INcCollection.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/INcCollection.java
deleted file mode 100644
index fc66008..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/INcCollection.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.hdfs.api;
-
-import org.apache.hadoop.mapred.InputSplit;
-
-@SuppressWarnings("deprecation")
-public interface INcCollection {
-
- public String findNearestAvailableSlot(InputSplit split);
-
- public int numAvailableSlots();
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/INcCollectionBuilder.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/INcCollectionBuilder.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/INcCollectionBuilder.java
deleted file mode 100644
index 02c5fb3..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/INcCollectionBuilder.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.hdfs.api;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hyracks.api.client.NodeControllerInfo;
-
-/**
- * NC collections
- *
- * @author yingyib
- */
-public interface INcCollectionBuilder {
-
- public INcCollection build(Map<String, NodeControllerInfo> ncNameToNcInfos, Map<String, List<String>> ipToNcMapping,
- Map<String, Integer> ncNameToIndex, String[] NCs, int[] workloads, int slotLimit);
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/ITupleWriter.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/ITupleWriter.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/ITupleWriter.java
deleted file mode 100644
index f950750..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/ITupleWriter.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.hdfs.api;
-
-import java.io.DataOutput;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-
-/**
- * Users need to implement this interface to use the HDFSWriteOperatorDescriptor.
- */
-public interface ITupleWriter {
-
- /**
- * Initialize the the tuple writer.
- *
- * @param output
- * The channel for output data.
- * @throws HyracksDataException
- */
- public void open(DataOutput output) throws HyracksDataException;
-
- /**
- * Write the tuple to the DataOutput.
- *
- * @param output
- * the DataOutput channel
- * @param tuple
- * the tuple to write
- * @throws HyracksDataException
- */
- public void write(DataOutput output, ITupleReference tuple) throws HyracksDataException;
-
- /**
- * Close the writer.
- *
- * @param output
- * The channel for output data.
- * @throws HyracksDataException
- */
- public void close(DataOutput output) throws HyracksDataException;
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/ITupleWriterFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/ITupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/ITupleWriterFactory.java
deleted file mode 100644
index 57dadb0..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/api/ITupleWriterFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.hdfs.api;
-
-import java.io.Serializable;
-
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-/**
- * Users need to implement this interface to use the HDFSWriteOperatorDescriptor.
- */
-public interface ITupleWriterFactory extends Serializable {
-
- /**
- * @param ctx
- * the IHyracksTaskContext
- * @return a tuple writer instance
- */
- public ITupleWriter getTupleWriter(IHyracksTaskContext ctx, int partition, int nPartition)
- throws HyracksDataException;
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/ConfFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/ConfFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/ConfFactory.java
deleted file mode 100644
index 33b84dc..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/ConfFactory.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.hdfs.dataflow;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.Serializable;
-
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class ConfFactory implements Serializable {
- private static final long serialVersionUID = 1L;
- private byte[] confBytes;
-
- public ConfFactory(JobConf conf) throws HyracksDataException {
- try {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- DataOutputStream dos = new DataOutputStream(bos);
- conf.write(dos);
- confBytes = bos.toByteArray();
- dos.close();
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- public synchronized JobConf getConf() throws HyracksDataException {
- try {
- JobConf conf = new JobConf();
- DataInputStream dis = new DataInputStream(new ByteArrayInputStream(confBytes));
- conf.readFields(dis);
- dis.close();
- return conf;
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSReadOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSReadOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSReadOperatorDescriptor.java
deleted file mode 100644
index 2aa45b7..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSReadOperatorDescriptor.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.hdfs.dataflow;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
-import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
-import org.apache.hyracks.hdfs.api.IKeyValueParser;
-import org.apache.hyracks.hdfs.api.IKeyValueParserFactory;
-
-/**
- * The HDFS file read operator using the Hadoop old API.
- * To use this operator, a user need to provide an IKeyValueParserFactory implementation which convert
- * key-value pairs into tuples.
- */
-@SuppressWarnings({ "rawtypes" })
-public class HDFSReadOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
-
- private static final long serialVersionUID = 1L;
- private final ConfFactory confFactory;
- private final InputSplitsFactory splitsFactory;
- private final String[] scheduledLocations;
- private final IKeyValueParserFactory tupleParserFactory;
- private final boolean[] executed;
-
- /**
- * The constructor of HDFSReadOperatorDescriptor.
- *
- * @param spec
- * the JobSpecification object
- * @param rd
- * the output record descriptor
- * @param conf
- * the Hadoop JobConf object, which contains the input format and the input paths
- * @param splits
- * the array of FileSplits (HDFS chunks).
- * @param scheduledLocations
- * the node controller names to scan the FileSplits, which is an one-to-one mapping. The String array
- * is obtained from the edu.cui.ics.hyracks.hdfs.scheduler.Scheduler.getLocationConstraints(InputSplits[]).
- * @param tupleParserFactory
- * the ITupleParserFactory implementation instance.
- * @throws HyracksException
- */
- public HDFSReadOperatorDescriptor(JobSpecification spec, RecordDescriptor rd, JobConf conf, InputSplit[] splits,
- String[] scheduledLocations, IKeyValueParserFactory tupleParserFactory) throws HyracksException {
- super(spec, 0, 1);
- try {
- this.splitsFactory = new InputSplitsFactory(splits);
- this.confFactory = new ConfFactory(conf);
- } catch (Exception e) {
- throw HyracksException.create(e);
- }
- this.scheduledLocations = scheduledLocations;
- this.executed = new boolean[scheduledLocations.length];
- Arrays.fill(executed, false);
- this.tupleParserFactory = tupleParserFactory;
- this.outRecDescs[0] = rd;
- }
-
- @Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
- throws HyracksDataException {
- final InputSplit[] inputSplits = splitsFactory.getSplits();
-
- return new AbstractUnaryOutputSourceOperatorNodePushable() {
- private String nodeName = ctx.getJobletContext().getServiceContext().getNodeId();
-
- @SuppressWarnings("unchecked")
- @Override
- public void initialize() throws HyracksDataException {
- ClassLoader ctxCL = Thread.currentThread().getContextClassLoader();
- try {
- writer.open();
- Thread.currentThread().setContextClassLoader(ctx.getJobletContext().getClassLoader());
- JobConf conf = confFactory.getConf();
- conf.setClassLoader(ctx.getJobletContext().getClassLoader());
- IKeyValueParser parser = tupleParserFactory.createKeyValueParser(ctx);
- try {
- parser.open(writer);
- InputFormat inputFormat = conf.getInputFormat();
- for (int i = 0; i < inputSplits.length; i++) {
- /**
- * read all the partitions scheduled to the current node
- */
- if (scheduledLocations[i].equals(nodeName)) {
- /**
- * pick an unread split to read
- * synchronize among simultaneous partitions in the same machine
- */
- synchronized (executed) {
- if (executed[i] == false) {
- executed[i] = true;
- } else {
- continue;
- }
- }
-
- /**
- * read the split
- */
- RecordReader reader = inputFormat.getRecordReader(inputSplits[i], conf, Reporter.NULL);
- Object key = reader.createKey();
- Object value = reader.createValue();
- while (reader.next(key, value) == true) {
- parser.parse(key, value, writer, inputSplits[i].toString());
- }
- }
- }
- } finally {
- parser.close(writer);
- }
- } catch (Throwable th) {
- writer.fail();
- throw HyracksDataException.create(th);
- } finally {
- writer.close();
- Thread.currentThread().setContextClassLoader(ctxCL);
- }
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSWriteOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSWriteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSWriteOperatorDescriptor.java
deleted file mode 100644
index 0babc5f..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSWriteOperatorDescriptor.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.hdfs.dataflow;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
-import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
-import org.apache.hyracks.hdfs.api.ITupleWriter;
-import org.apache.hyracks.hdfs.api.ITupleWriterFactory;
-
-/**
- * The HDFS file write operator using the Hadoop old API.
- * To use this operator, a user need to provide an ITupleWriterFactory.
- */
-@SuppressWarnings("deprecation")
-public class HDFSWriteOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
-
- private static final long serialVersionUID = 1L;
- private ConfFactory confFactory;
- private ITupleWriterFactory tupleWriterFactory;
-
- /**
- * The constructor of HDFSWriteOperatorDescriptor.
- *
- * @param spec
- * the JobSpecification object
- * @param conf
- * the Hadoop JobConf which contains the output path
- * @param tupleWriterFactory
- * the ITupleWriterFactory implementation object
- * @throws HyracksException
- */
- public HDFSWriteOperatorDescriptor(JobSpecification spec, JobConf conf, ITupleWriterFactory tupleWriterFactory)
- throws HyracksException {
- super(spec, 1, 0);
- this.confFactory = new ConfFactory(conf);
- this.tupleWriterFactory = tupleWriterFactory;
- }
-
- @Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
- throws HyracksDataException {
-
- return new AbstractUnaryInputSinkOperatorNodePushable() {
-
- private FSDataOutputStream dos;
- private RecordDescriptor inputRd = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);;
- private FrameTupleAccessor accessor = new FrameTupleAccessor(inputRd);
- private FrameTupleReference tuple = new FrameTupleReference();
- private ITupleWriter tupleWriter;
- private ClassLoader ctxCL;
-
- @Override
- public void open() throws HyracksDataException {
- ctxCL = Thread.currentThread().getContextClassLoader();
- Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
- JobConf conf = confFactory.getConf();
- String outputDirPath = FileOutputFormat.getOutputPath(conf).toString();
- String fileName = outputDirPath + File.separator + "part-" + partition;
-
- tupleWriter = tupleWriterFactory.getTupleWriter(ctx, partition, nPartitions);
- try {
- FileSystem dfs = FileSystem.get(conf);
- dos = dfs.create(new Path(fileName), true);
- tupleWriter.open(dos);
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- accessor.reset(buffer);
- int tupleCount = accessor.getTupleCount();
- for (int i = 0; i < tupleCount; i++) {
- tuple.reset(accessor, i);
- tupleWriter.write(dos, tuple);
- }
- }
-
- @Override
- public void fail() throws HyracksDataException {
-
- }
-
- @Override
- public void close() throws HyracksDataException {
- try {
- tupleWriter.close(dos);
- dos.close();
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- } finally {
- Thread.currentThread().setContextClassLoader(ctxCL);
- }
- }
-
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/InputSplitsFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/InputSplitsFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/InputSplitsFactory.java
deleted file mode 100644
index 4402964..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/InputSplitsFactory.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.hdfs.dataflow;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.Serializable;
-import java.lang.reflect.Constructor;
-
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-@SuppressWarnings({ "rawtypes" })
-public class InputSplitsFactory implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private byte[] splitBytes;
- private String splitClassName;
-
- public InputSplitsFactory(InputSplit[] splits) throws HyracksDataException {
- splitBytes = splitsToBytes(splits);
- if (splits.length > 0) {
- splitClassName = splits[0].getClass().getName();
- } else {
- splitClassName = FileSplit.class.getName();
- }
- }
-
- public InputSplit[] getSplits() throws HyracksDataException {
- return bytesToSplits(splitBytes);
- }
-
- /**
- * Convert splits to bytes.
- *
- * @param splits
- * input splits
- * @return bytes which serialize the splits
- * @throws IOException
- */
- private byte[] splitsToBytes(InputSplit[] splits) throws HyracksDataException {
- try {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- DataOutputStream dos = new DataOutputStream(bos);
- dos.writeInt(splits.length);
- for (int i = 0; i < splits.length; i++) {
- splits[i].write(dos);
- }
- dos.close();
- return bos.toByteArray();
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- /**
- * Covert bytes to splits.
- *
- * @param bytes
- * @return
- * @throws HyracksDataException
- */
- private InputSplit[] bytesToSplits(byte[] bytes) throws HyracksDataException {
- try {
- Class splitClass = Class.forName(splitClassName);
- Constructor[] constructors = splitClass.getDeclaredConstructors();
- Constructor defaultConstructor = null;
- for (Constructor constructor : constructors) {
- if (constructor.getParameterTypes().length == 0) {
- constructor.setAccessible(true);
- defaultConstructor = constructor;
- }
- }
- ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
- DataInputStream dis = new DataInputStream(bis);
- int size = dis.readInt();
- InputSplit[] splits = new InputSplit[size];
- for (int i = 0; i < size; i++) {
- splits[i] = (InputSplit) defaultConstructor.newInstance();
- splits[i].readFields(dis);
- }
- dis.close();
- return splits;
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/RawBinaryComparatorFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/RawBinaryComparatorFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/RawBinaryComparatorFactory.java
deleted file mode 100644
index 01acddd..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/RawBinaryComparatorFactory.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.hdfs.lib;
-
-import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
-import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-
-public class RawBinaryComparatorFactory implements IBinaryComparatorFactory {
-
- private static final long serialVersionUID = 1L;
- private static final IBinaryComparator comparator = RawBinaryComparatorFactory::compare;
- public static final IBinaryComparatorFactory INSTANCE = new RawBinaryComparatorFactory();
-
- private RawBinaryComparatorFactory() {
- }
-
- @Override
- public IBinaryComparator createBinaryComparator() {
- return comparator;
- }
-
- public static final int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- int commonLength = Math.min(l1, l2);
- for (int i = 0; i < commonLength; i++) {
- if (b1[s1 + i] != b2[s2 + i]) {
- return b1[s1 + i] - b2[s2 + i];
- }
- }
- int difference = l1 - l2;
- return difference == 0 ? 0 : (difference > 0 ? 1 : -1);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/RawBinaryHashFunctionFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/RawBinaryHashFunctionFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/RawBinaryHashFunctionFactory.java
deleted file mode 100644
index 047bbec..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/RawBinaryHashFunctionFactory.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.hdfs.lib;
-
-import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
-import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-
-public class RawBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
- private static final long serialVersionUID = 1L;
-
- public static IBinaryHashFunctionFactory INSTANCE = new RawBinaryHashFunctionFactory();
-
- private RawBinaryHashFunctionFactory() {
- }
-
- @Override
- public IBinaryHashFunction createBinaryHashFunction() {
-
- return new IBinaryHashFunction() {
- @Override
- public int hash(byte[] bytes, int offset, int length) {
- int value = 1;
- int end = offset + length;
- for (int i = offset; i < end; i++)
- value = value * 31 + (int) bytes[i];
- return value;
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2d90c0c1/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/TextKeyValueParserFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/TextKeyValueParserFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/TextKeyValueParserFactory.java
deleted file mode 100644
index f6bf66c..0000000
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/lib/TextKeyValueParserFactory.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.hdfs.lib;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
-import org.apache.hyracks.hdfs.api.IKeyValueParser;
-import org.apache.hyracks.hdfs.api.IKeyValueParserFactory;
-
-public class TextKeyValueParserFactory implements IKeyValueParserFactory<LongWritable, Text> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IKeyValueParser<LongWritable, Text> createKeyValueParser(final IHyracksTaskContext ctx)
- throws HyracksDataException {
-
- final ArrayTupleBuilder tb = new ArrayTupleBuilder(1);
- final FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
-
- return new IKeyValueParser<LongWritable, Text>() {
-
- @Override
- public void open(IFrameWriter writer) {
-
- }
-
- @Override
- public void parse(LongWritable key, Text value, IFrameWriter writer, String fileString)
- throws HyracksDataException {
- tb.reset();
- tb.addField(value.getBytes(), 0, value.getLength());
- FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- tb.getSize());
- }
-
- @Override
- public void close(IFrameWriter writer) throws HyracksDataException {
- appender.write(writer, false);
- }
-
- };
- }
-
-}