You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2017/03/28 23:43:52 UTC
bigtop git commit: BIGTOP-2727. fix deployment tasks
Repository: bigtop
Updated Branches:
refs/heads/master 738ff0746 -> ebcdedf3d
BIGTOP-2727. fix deployment tasks
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ebcdedf3
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ebcdedf3
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ebcdedf3
Branch: refs/heads/master
Commit: ebcdedf3dd24df38d8ed7b7ee547b11d8268db4e
Parents: 738ff07
Author: Roman Shaposhnik <rv...@apache.org>
Authored: Tue Mar 28 16:41:49 2017 -0700
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Tue Mar 28 16:43:39 2017 -0700
----------------------------------------------------------------------
bigtop-tests/build.gradle | 5 +
bigtop-tests/test-artifacts/fatjar/pom.xml | 11 +-
.../hbase/system/TestMoveRootMetaRegions.java | 150 ------------------
pom.xml | 151 ++++++-------------
release.gradle | 42 +++---
5 files changed, 80 insertions(+), 279 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/bigtop-tests/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/build.gradle b/bigtop-tests/build.gradle
index 2386d7e..f950fc0 100644
--- a/bigtop-tests/build.gradle
+++ b/bigtop-tests/build.gradle
@@ -57,6 +57,11 @@ distributions {
}
}
+tasks.withType(Tar) {
+ compression = Compression.GZIP
+ extension = "tar.gz"
+}
+
distZip.dependsOn clean
distTar.dependsOn clean
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/bigtop-tests/test-artifacts/fatjar/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/fatjar/pom.xml b/bigtop-tests/test-artifacts/fatjar/pom.xml
index ca01eda..ba0a942 100644
--- a/bigtop-tests/test-artifacts/fatjar/pom.xml
+++ b/bigtop-tests/test-artifacts/fatjar/pom.xml
@@ -128,19 +128,16 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.0.0</version>
<configuration>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
+ <!-- put your configurations here -->
</configuration>
<executions>
<execution>
- <id>make-assembly</id>
<phase>package</phase>
<goals>
- <goal>assembly</goal>
+ <goal>shade</goal>
</goals>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java
deleted file mode 100644
index 2132a43..0000000
--- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.bigtop.itest.hbase.system;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.not;
-
-import org.apache.bigtop.itest.shell.Shell;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * This test contains 3 tests:
- * <p/>
- * 1) Unload the region server hosting -ROOT-. Check that -ROOT- region
- * is accessible after a configurable period of time.
- * <p/>
- * 2) The test above for the .META. region.
- * <p/>
- * 3) Stop the region server(s) hosting the -ROOT- and .META. tables
- * and verify the regions are moved to other region server(s).
- * <p/>
- * This test does not restart the stopped region server, so users will
- * have to manually restart the region server.
- * <p/>
- * The third test is designed for clusters with more than two region servers.
- * <p/>
- * Optional arguments:
- * -Droot.timeout.ms=<milliseconds to wait while trying to find -ROOT->
- * -Dmeta.timeout.ms=<milliseconds to wait while trying to find .META.>
- * -Dwait.after.move.ms=<milliseconds to wait after moving -ROOT- or .META.>
- */
-
-public class TestMoveRootMetaRegions {
- private static final String HBASE_HOME = System.getenv("HBASE_HOME");
-
- static {
- Assert.assertNotNull("HBASE_HOME has to be set to run this test", HBASE_HOME);
- }
-
- private static Shell sh = new Shell("/bin/bash -s");
-
- private static String load_regionserver =
- "$HBASE_HOME/bin/hbase org.jruby.Main $HBASE_HOME/bin/region_mover.rb load ";
- private static String unload_regionserver =
- "$HBASE_HOME/bin/hbase org.jruby.Main $HBASE_HOME/bin/region_mover.rb unload ";
-
- private static long meta_timeout_ms;
- private static long root_timeout_ms;
- private static long wait_after_move_ms;
-
- private static Configuration conf;
- private static HBaseAdmin admin;
- private static CatalogTracker ct;
-
- private static String meta_table =
- Bytes.toStringBinary(TableName.META_TABLE_NAME.getName());
-
- @BeforeClass
- public static void setUp() throws Exception {
- // Default timeout is 3 minutes.
- root_timeout_ms =
- Integer.parseInt(System.getProperty("root.timeout.ms", "180000"));
- meta_timeout_ms =
- Integer.parseInt(System.getProperty("meta.timeout.ms", "180000"));
- // Default to 20 seconds.
- wait_after_move_ms =
- Integer.parseInt(System.getProperty("wait.after.move.ms", "20000"));
-
- conf = HBaseConfiguration.create();
- admin = new HBaseAdmin(conf);
-
- ct = new CatalogTracker(admin.getConfiguration());
- ct.start();
-
- Assert.assertTrue(admin.tableExists(HConstants.META_TABLE_NAME));
- Assert.assertTrue(admin.isTableEnabled(HConstants.META_TABLE_NAME));
- }
-
- @AfterClass
- public static void tearDown() {
- // Cleanup
- ct.stop();
- }
-
- public static ServerName getMetaAddress() throws Exception {
- return ct.waitForMeta(meta_timeout_ms);
- }
-
- @Test
- public void unloadMetaRegionServer() throws Exception {
- ServerName meta_address = getMetaAddress();
- String cmd = unload_regionserver + meta_address.getHostname();
- System.out.println("Unloading the region server hosting " + meta_table);
- System.out.println(cmd);
- sh.exec(cmd);
-
- Thread.sleep(wait_after_move_ms);
- getMetaAddress();
-
- cmd = load_regionserver + meta_address.getHostname();
- System.out.println("Reloading the region server");
- sh.exec(cmd);
- Thread.sleep(wait_after_move_ms);
- }
-
- @Test
- public void testStopRootMetaRegionServers() throws Exception {
- ServerName meta_address = getMetaAddress();
-
- boolean same_server = false;
-
- System.out.println(meta_table + " server address: " + meta_address);
-
- System.out.println("Stopping region server(s)");
- admin.stopRegionServer(meta_address.getHostAndPort());
-
- Thread.sleep(wait_after_move_ms);
-
- ServerName new_meta_address = getMetaAddress();
-
- System.out.println(meta_table + " server address: " + new_meta_address);
- Assert.assertThat(meta_address, not(equalTo(new_meta_address)));
- }
-}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f128314..5c5de30 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,19 +39,19 @@
<inceptionYear>2011</inceptionYear>
<properties>
- <hadoop.version>2.7.2</hadoop.version>
+ <hadoop.version>2.7.3</hadoop.version>
<!--An awful hack for BIGTOP-1429-->
<hbase.version>1.1.3</hbase.version>
- <pig.version>0.12.1</pig.version>
- <pig-smoke.version>0.12.1</pig-smoke.version>
- <sqoop.version>1.4.5</sqoop.version>
+ <pig.version>0.15.0</pig.version>
+ <pig-smoke.version>0.15.0</pig-smoke.version>
+ <sqoop.version>1.4.6</sqoop.version>
<sqoop2.version>1.99.4</sqoop2.version>
- <zookeeper.version>3.4.5</zookeeper.version>
- <giraph.version>1.0.0</giraph.version>
- <solr.version>4.6.0</solr.version>
- <spark.version>1.3.1</spark.version>
- <kafka.version>0.8.1.1</kafka.version>
- <phoenix.version>4.7.0</phoenix.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
+ <giraph.version>1.1.0</giraph.version>
+ <solr.version>4.10.4</solr.version>
+ <spark.version>2.1.0</spark.version>
+ <kafka.version>0.10.1.1</kafka.version>
+ <phoenix.version>4.9.0-HBase-1.1</phoenix.version>
<kite.version>1.1.0</kite.version>
<spark-smoke.version>${project.version}</spark-smoke.version>
@@ -290,86 +290,6 @@
<pluginManagement>
<plugins>
<plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>0.7</version>
- <configuration>
- <excludes>
- <exclude>.git/**</exclude>
- <exclude>.idea/**</exclude>
- <exclude>**/.gitignore</exclude>
- <!-- Test data with rigid structure and/or binary -->
- <exclude>bigtop-tests/test-artifacts/**/resources/**</exclude>
- <exclude>bigtop-tests/smoke-tests/alluxio/datafile</exclude>
- <exclude>bigtop-tests/smoke-tests/hive/passwd.ql</exclude>
- <exclude>bigtop-tests/smoke-tests/kite/sandwiches.csv</exclude>
- <exclude>bigtop-tests/smoke-tests/phoenix/*.csv</exclude>
- <exclude>bigtop-tests/smoke-tests/phoenix/*.sql</exclude>
- <exclude>bigtop-tests/smoke-tests/ignite-hadoop/*.data</exclude>
- <exclude>bigtop-tests/smoke-tests/tajo/table1/*.csv</exclude>
- <exclude>**/target/**</exclude>
- <exclude>**/build/**</exclude>
- <exclude>**/.gradle/**</exclude>
- <exclude>**/*.iml</exclude>
- <exclude>.gradle</exclude>
- <exclude>output/**</exclude>
- <exclude>MAINTAINERS.txt</exclude>
- <exclude>buildSrc/build/**</exclude>
- <exclude>bigtop-data-generators/**/input_data/**</exclude>
- <exclude>docker/**/*.yaml</exclude>
- <exclude>bigtop-deploy/puppet/**/*.yaml</exclude>
- <exclude>bigtop-deploy/puppet/modules/hadoop/**/id_hdfsuser*</exclude>
- <exclude>bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock</exclude>
- <exclude>bigtop-deploy/puppet/modules/zeppelin/templates/interpreter.json</exclude>
- <!-- Debian package files that have a rigid structure (* matches package name) -->
- <exclude>bigtop-packages/src/deb/*/*.dirs</exclude>
- <exclude>bigtop-packages/src/deb/*/dirs</exclude>
- <exclude>bigtop-packages/src/deb/*/*.install</exclude>
- <exclude>bigtop-packages/src/deb/*/*.include</exclude>
- <exclude>bigtop-packages/src/deb/**/include-binaries</exclude>
- <exclude>bigtop-packages/src/deb/**/format</exclude>
- <exclude>bigtop-packages/src/deb/*/*.manpages</exclude>
- <exclude>bigtop-packages/src/deb/*/*.docs</exclude>
- <exclude>bigtop-packages/src/deb/*/docs</exclude>
- <exclude>bigtop-packages/src/deb/*/compat</exclude>
- <exclude>bigtop-packages/src/deb/*/source/format</exclude>
- <exclude>bigtop-packages/src/deb/*/*.lintian-overrides</exclude>
- <exclude>bigtop-packages/src/deb/*/shlibs.local</exclude>
- <exclude>bigtop-packages/src/**/*.diff</exclude>
- <exclude>bigtop-packages/src/common/*/*.json</exclude>
- <exclude>bigtop-packages/src/common/**/*.default</exclude>
- <exclude>bigtop-repos/apt/distributions</exclude>
- <!-- Juju charm files with rigid structure -->
- <exclude>bigtop-packages/src/charm/**/wheelhouse.txt</exclude>
- <exclude>bigtop-packages/src/charm/**/*.json</exclude>
- <exclude>bigtop-packages/src/charm/**/*.svg</exclude>
- <exclude>bigtop-packages/src/charm/**/*.yaml</exclude>
- <exclude>bigtop-deploy/juju/**/*.yaml</exclude>
- <exclude>bigtop-tests/cloud-weather-report/**/*.yaml</exclude>
- <!-- Miscelaneous individual files -->
- <exclude>src/site/resources/bigtop.rdf</exclude>
- <exclude>src/site/resources/images/bigtop-logo.ai</exclude>
- <!--Excluding a file with FreeBSD Documentation License-->
- <exclude>src/site/resources/css/freebsd_docbook.css</exclude>
- <exclude>test/MANIFEST.txt</exclude>
- <exclude>
- bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
- </exclude>
- <exclude>
- bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg
- </exclude>
- <exclude>bigtop-deploy/puppet/README.md</exclude>
- <exclude>bigtop-deploy/puppet/modules/qfs/README.md</exclude>
- <exclude>bigtop-deploy/puppet/config/site.csv.example</exclude>
- <exclude>dl/**</exclude>
- <exclude>bigtop-deploy/puppet/modules/gpdb/templates/**</exclude>
- <exclude>provisioner/docker/config/hiera.yaml</exclude>
- <exclude>provisioner/docker/config/hieradata/PLACEHOLDER</exclude>
- <exclude>provisioner/docker/config/hosts</exclude>
- </excludes>
- </configuration>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-pdf-plugin</artifactId>
<version>1.2</version>
@@ -399,6 +319,11 @@
<artifactId>maven-scm-publish-plugin</artifactId>
<version>1.0</version>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>3.0.0</version>
+ </plugin>
</plugins>
</pluginManagement>
<plugins>
@@ -488,6 +413,40 @@
<profiles>
<profile>
+ <id>attach-smokes</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>bigtop-tests/build/distributions/bigtop-smokes-suite-${project.version}.tar.gz</file>
+ <type>tar.gz</type>
+ <classifier>testsuite</classifier>
+ </artifact>
+ <artifact>
+ <file>bigtop-tests/build/distributions/bigtop-smokes-suite-${project.version}.zip</file>
+ <type>zip</type>
+ <classifier>testsuite</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>release</id>
<build>
<plugins>
@@ -509,18 +468,6 @@
</executions>
</plugin>
<plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<executions>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/release.gradle
----------------------------------------------------------------------
diff --git a/release.gradle b/release.gradle
index 7d1e5f5..6b9afca 100644
--- a/release.gradle
+++ b/release.gradle
@@ -16,45 +16,47 @@
* limitations under the License.
*
*/
-
apply plugin: 'groovy'
def final mvnProfile="release"
-task dosite (type: Exec) << {
+task dosite (type: Exec) {
workingDir "."
- commandLine 'mvn clean site'.split(" ")
+ commandLine "mvn clean site".split(" ")
}
-task doassembly (type: Exec) << {
+task doassembly (type: Exec) {
workingDir "."
- commandLine 'mvn -P$mvnProfile package assembly:assembly'.split(" ")
+ commandLine "mvn -P$mvnProfile package assembly:assembly".split(" ")
}
-task deployTop (type: Exec) << {
+task deployTop (type: Exec, dependsOn: ["bigtop-tests:assembleDist"]) {
workingDir "."
- commandLine 'mvn -P$mvnProfile deploy -f pom.xml'.split(" ")
+ commandLine "mvn -P$mvnProfile -Pattach-smokes deploy -f pom.xml".split(" ")
+ standardInput System.in
}
-task deployITest (type: Exec) << {
+task deployITest (type: Exec) {
workingDir "."
- commandLine 'mvn -P$mvnProfile deploy -f bigtop-test-framework/pom.xml'.split(" ")
+ commandLine "mvn -P$mvnProfile -DskipTests deploy -f bigtop-test-framework/pom.xml".split(" ")
+ standardInput System.in
}
-task deployTestArtifacts (type: Exec) << {
+task deployTestArtifacts (type: Exec) {
workingDir "."
- commandLine 'mvn -P$mvnProfile deploy -f bigtop-tests/test-artifacts/pom.xml'.split(" ")
+ commandLine "mvn -P$mvnProfile deploy -f bigtop-tests/test-artifacts/pom.xml".split(" ")
+ standardInput System.in
}
-task deployTestExecutors (type: Exec) << {
+task deployTestExecutors (type: Exec) {
workingDir "."
- commandLine 'mvn -P$mvnProfile deploy -f bigtop-tests/test-execution/pom.xml'.split(" ")
+ commandLine "mvn -P$mvnProfile deploy -f bigtop-tests/test-execution/pom.xml".split(" ")
+ environment HADOOP_CONF_DIR: "", HADOOP_HOME: ""
+ standardInput System.in
}
-task deployAll (type: Exec,
- dependsOn: [deployTestExecutors, deployTestArtifacts,
- deployITest, deployTop]) << {
- workingDir "."
- commandLine 'mvn deploy -Prelease'.split(" ")
+task deployAll (dependsOn: [deployTestExecutors, deployTestArtifacts,
+ deployITest, deployTop]) << {
+ println "Done!"
}
task dorelease(description: 'Convenient wrapper for existing release tasks',
- dependsOn: [doassembly, dosite, deployAll]) << {
+ dependsOn: [dosite, deployAll]) << {
}
task setversion << {
@@ -67,4 +69,4 @@ task setversion << {
pomFile.write(pomFile.text.replaceAll(version, nextversion))
}
println "Done! Please inspect changes to make sure everything is in order."
-}
\ No newline at end of file
+}