You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2011/11/08 00:39:42 UTC
svn commit: r1199014 - in /incubator/bigtop/branches/hadoop-0.23:
bigtop-packages/src/common/sqoop/patch bigtop-packages/src/deb/sqoop/rules
bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec bigtop.mk
Author: rvs
Date: Mon Nov 7 23:39:41 2011
New Revision: 1199014
URL: http://svn.apache.org/viewvc?rev=1199014&view=rev
Log:
Making sqoop builds against .23
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec
incubator/bigtop/branches/hadoop-0.23/bigtop.mk
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch?rev=1199014&r1=1199013&r2=1199014&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/sqoop/patch Mon Nov 7 23:39:41 2011
@@ -1,43 +1,252 @@
-Index: sqoop-1.3.0/ivy.xml
-===================================================================
---- sqoop-1.3.0.orig/ivy.xml 2011-09-27 10:28:59.474837063 -0700
-+++ sqoop-1.3.0/ivy.xml 2011-09-27 10:28:46.733590903 -0700
-@@ -66,10 +66,18 @@
+diff --git ivy.xml ivy.xml
+index d686c76..a882dae 100644
+--- ivy.xml
++++ ivy.xml
+@@ -67,12 +67,19 @@ under the License.
+ <artifact conf="master"/>
</publications>
<dependencies>
- <!-- Dependencies for Cloudera's Distribution for Hadoop -->
-- <dependency org="com.cloudera.hadoop" name="hadoop-core"
+- <!-- Dependencies for Cloudera's Distribution for Hadoop -->
+- <dependency org="org.apache.hadoop" name="hadoop-core"
- rev="${hadoop-core.cloudera.version}" conf="cloudera->default"/>
-- <dependency org="com.cloudera.hadoop" name="hadoop-test"
+- <dependency org="org.apache.hadoop" name="hadoop-test"
- rev="${hadoop-core.cloudera.version}" conf="clouderatest->default"/>
-+ <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}"
+-
++ <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop.version}"
+ conf="cloudera->default"/>
-+ <dependency org="org.apache.hadoop" name="hadoop-common-test" rev="${hadoop-common.version}"
-+ conf="clouderatest->default"/>
-+ <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop-hdfs.version}"
++ <dependency org="org.apache.hadoop" name="hadoop-common-test" rev="${hadoop.version}"
++ conf="clouderatest->default"/>
++ <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop.version}"
+ conf="cloudera->default"/>
-+ <dependency org="org.apache.hadoop" name="hadoop-hdfs-test" rev="${hadoop-hdfs.version}"
++ <dependency org="org.apache.hadoop" name="hadoop-hdfs-test" rev="${hadoop.version}"
+ conf="clouderatest->default"/>
-+ <dependency org="org.apache.hadoop" name="hadoop-mapred" rev="${hadoop-mapred.version}"
++ <dependency org="org.apache.hadoop" name="hadoop-mapred" rev="${hadoop.version}"
+ conf="cloudera->default"/>
-+ <dependency org="org.apache.hadoop" name="hadoop-mapred-test" rev="${hadoop-mapred.version}"
++ <dependency org="org.apache.hadoop" name="hadoop-mapred-test" rev="${hadoop.version}"
+ conf="clouderatest->default"/>
-
-
++
<!-- Common dependencies for Sqoop -->
<dependency org="commons-cli" name="commons-cli"
-Index: sqoop-1.3.0/ivy/libraries.properties
-===================================================================
---- sqoop-1.3.0.orig/ivy/libraries.properties 2011-09-27 10:30:10.136087155 -0700
-+++ sqoop-1.3.0/ivy/libraries.properties 2011-09-27 10:29:29.734839438 -0700
-@@ -27,7 +27,9 @@
+ rev="${commons-cli.version}" conf="common->default"/>
+@@ -121,9 +128,13 @@ under the License.
+ <exclude org="com.sun.jersey" module="jersey-server"/>
+ <exclude org="org.apache.thrift" module="thrift"/>
+ <exclude org="log4j" module="log4j"/>
++ <exclude org="org.apache.hadoop" module="hadoop-core" />
++ <exclude org="com.cloudera.cdh" module="zookeeper-ant" />
+ </dependency>
+
+ <exclude org="org.apache.hadoop" module="avro"/>
++ <exclude org="commons-configuration" module="commons-configuration" />
++ <exclude org="commons-daemon" module="commons-daemon" />
+
+ </dependencies>
+ </ivy-module>
+diff --git ivy/libraries.properties b/ivy/libraries.properties
+index dbbcb3b..8ba9a8b 100644
+--- ivy/libraries.properties
++++ ivy/libraries.properties
+@@ -29,14 +29,14 @@ commons-lang.version=2.4
commons-logging.version=1.0.4
# Cloudera Distribution dependency version
--hadoop-core.cloudera.version=0.20.2-737
-+hadoop-common.version=0.23.0-SNAPSHOT
-+hadoop-hdfs.version=0.23.0-SNAPSHOT
-+hadoop-mapred.version=0.23.0-SNAPSHOT
+-hadoop-core.cloudera.version=0.20.2-cdh3u1
++hadoop.version=0.23.0-SNAPSHOT
+
+ hbase.version=0.90.3-cdh3u1
+ zookeeper.version=3.3.3-cdh3u1
+
+ hsqldb.version=1.8.0.10
+
+-ivy.version=2.0.0-rc2
++ivy.version=2.1.0
+
+ junit.version=4.5
+
+Index: src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java
+===================================================================
+--- src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java (revision 1198954)
++++ src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java (working copy)
+@@ -31,10 +31,7 @@
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.FileSystem;
+ import org.apache.hadoop.fs.Path;
+-import org.apache.hadoop.mapreduce.MapContext;
+-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+-import com.cloudera.sqoop.testutil.MockObjectFactory;
+ import com.cloudera.sqoop.testutil.MockResultSet;
+
+ /**
+@@ -45,7 +42,6 @@
+ protected Configuration conf;
+ protected LargeObjectLoader loader;
+ protected Path outDir;
+- protected MapContext mapContext;
+
+ public void setUp() throws IOException, InterruptedException {
+ conf = new Configuration();
+@@ -60,17 +56,7 @@
+ }
+ fs.mkdirs(outDir);
+
+- /* A mock MapContext that uses FileOutputCommitter.
+- * This MapContext is actually serving two roles here; when writing the
+- * CLOB files, its OutputCommitter is used to determine where to write
+- * the CLOB data, as these are placed in the task output work directory.
+- * When reading the CLOB data back for verification, we use the
+- * getInputSplit() to determine where to read our source data from--the same
+- * directory. We are repurposing the same context for both output and input.
+- */
+- mapContext = MockObjectFactory.getMapContextForIOPath(conf, outDir);
+- loader = new LargeObjectLoader(mapContext.getConfiguration(),
+- FileOutputFormat.getWorkOutputPath(mapContext));
++ loader = new LargeObjectLoader(conf, outDir);
+ }
+
+ public void testReadClobRef()
+@@ -88,7 +74,6 @@
+ assertNotNull(clob);
+ assertTrue(clob.isExternal());
+ loader.close();
+- mapContext.getOutputCommitter().commitTask(mapContext);
+ Reader r = clob.getDataStream(conf, outDir);
+ char [] buf = new char[4096];
+ int chars = r.read(buf, 0, 4096);
+@@ -117,7 +102,6 @@
+ assertNotNull(blob);
+ assertTrue(blob.isExternal());
+ loader.close();
+- mapContext.getOutputCommitter().commitTask(mapContext);
+ InputStream is = blob.getDataStream(conf, outDir);
+ byte [] buf = new byte[4096];
+ int bytes = is.read(buf, 0, 4096);
+Index: src/test/com/cloudera/sqoop/testutil/MockObjectFactory.java
+===================================================================
+--- src/test/com/cloudera/sqoop/testutil/MockObjectFactory.java (revision 1198954)
++++ src/test/com/cloudera/sqoop/testutil/MockObjectFactory.java (working copy)
+@@ -1,84 +0,0 @@
+-/**
+- * Licensed to the Apache Software Foundation (ASF) under one
+- * or more contributor license agreements. See the NOTICE file
+- * distributed with this work for additional information
+- * regarding copyright ownership. The ASF licenses this file
+- * to you under the Apache License, Version 2.0 (the
+- * "License"); you may not use this file except in compliance
+- * with the License. You may obtain a copy of the License at
+- *
+- * http://www.apache.org/licenses/LICENSE-2.0
+- *
+- * Unless required by applicable law or agreed to in writing, software
+- * distributed under the License is distributed on an "AS IS" BASIS,
+- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+- * See the License for the specific language governing permissions and
+- * limitations under the License.
+- */
+-
+-package com.cloudera.sqoop.testutil;
+-
+-import java.io.IOException;
+-
+-import org.apache.hadoop.conf.Configuration;
+-import org.apache.hadoop.fs.Path;
+-import org.apache.hadoop.mapreduce.Counters;
+-import org.apache.hadoop.mapreduce.InputSplit;
+-import org.apache.hadoop.mapreduce.MapContext;
+-import org.apache.hadoop.mapreduce.OutputCommitter;
+-import org.apache.hadoop.mapreduce.TaskAttemptID;
+-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+-import org.apache.hadoop.mrunit.mapreduce.mock.MockReporter;
+-
+-/**
+- * Allows the creation of various mock objects for testing purposes.
+- */
+-public final class MockObjectFactory {
+-
+- /**
+- * Returns a mock MapContext that has both an OutputCommitter and an
+- * InputSplit wired to the specified path.
+- * Used for testing LargeObjectLoader.
+- */
+- public static MapContext getMapContextForIOPath(Configuration conf, Path p) {
+- return new MockMapContextWithCommitter(conf, p);
+- }
+-
+- private static class MockMapContextWithCommitter
+- extends MapContext<Object, Object, Object, Object> {
+- private Path path;
+- private Configuration conf;
+-
+- public MockMapContextWithCommitter(Configuration c, Path p) {
+- super(c, new TaskAttemptID("jt", 0, true, 0, 0),
+- null, null, null, new MockReporter(new Counters()), null);
+-
+- this.path = p;
+- this.conf = c;
+- }
+-
+- @Override
+- public OutputCommitter getOutputCommitter() {
+- try {
+- return new FileOutputCommitter(path, this);
+- } catch (IOException ioe) {
+- return null;
+- }
+- }
+-
+- @Override
+- public InputSplit getInputSplit() {
+- return new FileSplit(new Path(path, "inputFile"), 0, 0, new String[0]);
+- }
+-
+- @Override
+- public Configuration getConfiguration() {
+- return conf;
+- }
+- }
+-
+- private MockObjectFactory() {
+- // Disable explicity object creation
+- }
+-}
+Index: src/test/com/cloudera/sqoop/mapreduce/db/TestDataDrivenDBInputFormat.java
+===================================================================
+--- src/test/com/cloudera/sqoop/mapreduce/db/TestDataDrivenDBInputFormat.java (revision 1198954)
++++ src/test/com/cloudera/sqoop/mapreduce/db/TestDataDrivenDBInputFormat.java (working copy)
+@@ -23,13 +23,14 @@
+ import java.io.DataOutput;
+ import java.io.IOException;
+
++import junit.framework.TestCase;
++
+ import org.apache.commons.logging.Log;
+ import org.apache.commons.logging.LogFactory;
+ import org.apache.hadoop.fs.*;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.io.NullWritable;
+ import org.apache.hadoop.io.WritableComparable;
+-import org.apache.hadoop.mapred.HadoopTestCase;
+ import org.apache.hadoop.mapreduce.*;
+ import org.apache.hadoop.mapreduce.lib.db.*;
+ import org.apache.hadoop.mapreduce.lib.output.*;
+@@ -38,7 +39,7 @@
+ /**
+ * Test aspects of DataDrivenDBInputFormat.
+ */
+-public class TestDataDrivenDBInputFormat extends HadoopTestCase {
++public class TestDataDrivenDBInputFormat extends TestCase {
+
+ private static final Log LOG = LogFactory.getLog(
+ TestDataDrivenDBInputFormat.class);
+@@ -52,10 +53,6 @@
+
+ private static final String OUT_DIR;
- hbase.version=0.89.20100924-28
- zookeeper.version=3.3.1+7
+- public TestDataDrivenDBInputFormat() throws IOException {
+- super(LOCAL_MR, LOCAL_FS, 1, 1);
+- }
+-
+ static {
+ OUT_DIR = System.getProperty("test.build.data", "/tmp") + "/dddbifout";
+ }
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules?rev=1199014&r1=1199013&r2=1199014&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/sqoop/rules Mon Nov 7 23:39:41 2011
@@ -25,7 +25,7 @@ export DH_OPTIONS
patch: patch-stamp
patch-stamp:
- patch -p1 < debian/patch
+ patch -p0 < debian/patch
touch $@
clean:
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec?rev=1199014&r1=1199013&r2=1199014&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec Mon Nov 7 23:39:41 2011
@@ -79,8 +79,8 @@ Shared metadata repository for Sqoop. Th
server for Sqoop clients across a network to use.
%prep
-%setup -n sqoop-%{sqoop_base_version}
-%patch0 -p1
+%setup -n apache-sqoop-e9d0be
+%patch0 -p0
%build
bash %{SOURCE1} -Dversion=%{version}
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop.mk
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop.mk?rev=1199014&r1=1199013&r2=1199014&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop.mk (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop.mk Mon Nov 7 23:39:41 2011
@@ -86,12 +86,14 @@ $(eval $(call PACKAGE,hive,HIVE))
SQOOP_NAME=sqoop
SQOOP_RELNOTES_NAME=Sqoop
SQOOP_PKG_NAME=sqoop
-SQOOP_BASE_VERSION=1.3.0
-SQOOP_PKG_VERSION=1.3.0
+SQOOP_BASE_VERSION=1.4.0
+SQOOP_PKG_VERSION=1.4.0
SQOOP_RELEASE_VERSION=1
SQOOP_TARBALL_DST=sqoop-$(SQOOP_BASE_VERSION).tar.gz
-SQOOP_TARBALL_SRC=$(SQOOP_TARBALL_DST)
-SQOOP_SITE=http://github.com/downloads/cloudera/sqoop/
+#SQOOP_TARBALL_SRC=$(SQOOP_TARBALL_DST)
+#SQOOP_SITE=http://github.com/downloads/cloudera/sqoop/
+SQOOP_SITE=https://github.com/apache/sqoop/tarball
+SQOOP_TARBALL_SRC=e9d0bed
$(eval $(call PACKAGE,sqoop,SQOOP))
# Oozie