You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ju...@apache.org on 2018/09/17 04:41:34 UTC
[8/9] bigtop git commit: BIGTOP-3074: Drop oozie packaging
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-packages/src/rpm/oozie/SPECS/oozie.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/oozie/SPECS/oozie.spec b/bigtop-packages/src/rpm/oozie/SPECS/oozie.spec
deleted file mode 100644
index 933481a..0000000
--- a/bigtop-packages/src/rpm/oozie/SPECS/oozie.spec
+++ /dev/null
@@ -1,232 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-%define usr_bin /usr/bin
-%define lib_oozie /usr/lib/oozie
-%define man_dir /usr/share/man
-%define conf_oozie %{_sysconfdir}/%{name}/conf
-%define conf_oozie_dist %{conf_oozie}.dist
-%define tomcat_conf_oozie %{_sysconfdir}/%{name}/tomcat-conf
-%define data_oozie /var/lib/oozie
-%define lib_hadoop /usr/lib/hadoop
-
-%if %{!?suse_version:1}0
- %define doc_oozie %{_docdir}/oozie-%{oozie_version}
- %define initd_dir %{_sysconfdir}/rc.d/init.d
- %define alternatives_cmd alternatives
-%else
-
-# Only tested on openSUSE 11.4. le'ts update it for previous release when confirmed
-%if 0%{suse_version} > 1130
-%define suse_check \# Define an empty suse_check for compatibility with older sles
-%endif
-
-# SLES is more strict anc check all symlinks point to valid path
-# But we do point to a hadoop jar which is not there at build time
-# (but would be at install time).
-# Since our package build system does not handle dependencies,
-# these symlink checks are deactivated
-%define __os_install_post \
- %{suse_check} ; \
- /usr/lib/rpm/brp-compress ; \
- %{nil}
-
- %define doc_oozie %{_docdir}/oozie
- %define initd_dir %{_sysconfdir}/rc.d
- %define alternatives_cmd update-alternatives
-%endif
-
-Name: oozie
-Version: %{oozie_version}
-Release: %{oozie_release}
-Summary: Oozie is a system that runs workflows of Hadoop jobs.
-URL: http://oozie.apache.org/
-Group: Development/Libraries
-Buildroot: %{_topdir}/INSTALL/%{name}-%{version}
-License: ASL 2.0
-Source0: %{name}-%{oozie_base_version}.tar.gz
-Source1: do-component-build
-Source2: install_oozie.sh
-Source3: oozie.1
-Source4: oozie-env.sh
-Source5: oozie.init
-Source6: catalina.properties
-Source7: context.xml
-Source8: hive.xml
-Source9: tomcat-deployment.sh
-Source10: oozie-site.xml
-Source11: bigtop.bom
-#BIGTOP_PATCH_FILES
-Requires(pre): /usr/sbin/groupadd, /usr/sbin/useradd
-Requires(post): /sbin/chkconfig
-Requires(preun): /sbin/chkconfig, /sbin/service
-Requires: oozie-client = %{version}, hadoop-client, bigtop-tomcat
-BuildArch: noarch
-
-%description
- Oozie is a system that runs workflows of Hadoop jobs.
- Oozie workflows are actions arranged in a control dependency DAG (Direct
- Acyclic Graph).
-
- Oozie coordinator functionality allows to start workflows at regular
- frequencies and when data becomes available in HDFS.
-
- An Oozie workflow may contain the following types of actions nodes:
- map-reduce, map-reduce streaming, map-reduce pipes, pig, file-system,
- sub-workflows, java, hive, sqoop and ssh (deprecated).
-
- Flow control operations within the workflow can be done using decision,
- fork and join nodes. Cycles in workflows are not supported.
-
- Actions and decisions can be parameterized with job properties, actions
- output (i.e. Hadoop counters) and HDFS file information (file exists,
- file size, etc). Formal parameters are expressed in the workflow definition
- as ${VARIABLE NAME} variables.
-
- A Workflow application is an HDFS directory that contains the workflow
- definition (an XML file), all the necessary files to run all the actions:
- JAR files for Map/Reduce jobs, shells for streaming Map/Reduce jobs, native
- libraries, Pig scripts, and other resource files.
-
- Running workflow jobs is done via command line tools, a WebServices API
- or a Java API.
-
- Monitoring the system and workflow jobs can be done via a web console, the
- command line tools, the WebServices API and the Java API.
-
- Oozie is a transactional system and it has built in automatic and manual
- retry capabilities.
-
- In case of workflow job failure, the workflow job can be rerun skipping
- previously completed actions, the workflow application can be patched before
- being rerun.
-
-
-%package client
-Version: %{version}
-Release: %{release}
-Summary: Client for Oozie Workflow Engine
-URL: http://oozie.apache.org/
-Group: Development/Libraries
-License: ASL 2.0
-BuildArch: noarch
-Requires: bigtop-utils >= 0.7
-
-
-%description client
- Oozie client is a command line client utility that allows remote
- administration and monitoring of worflows. Using this client utility
- you can submit worflows, start/suspend/resume/kill workflows and
- find out their status at any instance. Apart from such operations,
- you can also change the status of the entire system, get vesion
- information. This client utility also allows you to validate
- any worflows before they are deployed to the Oozie server.
-
-
-%prep
-%setup -n oozie-%{oozie_base_version}
-
-#BIGTOP_PATCH_COMMANDS
-
-%build
- mkdir -p distro/downloads
- env DO_MAVEN_DEPLOY="" FULL_VERSION=%{oozie_base_version} bash -x %{SOURCE1}
-
-%install
-%__rm -rf $RPM_BUILD_ROOT
- sh %{SOURCE2} --extra-dir=$RPM_SOURCE_DIR --build-dir=$PWD --server-dir=$RPM_BUILD_ROOT --client-dir=$RPM_BUILD_ROOT --docs-dir=$RPM_BUILD_ROOT%{doc_oozie} --initd-dir=$RPM_BUILD_ROOT%{initd_dir} --conf-dir=$RPM_BUILD_ROOT%{conf_oozie_dist}
-
-%__ln_s -f %{data_oozie}/ext-2.2 $RPM_BUILD_ROOT/%{lib_oozie}/webapps/oozie/ext-2.2
-%__rm -rf $RPM_BUILD_ROOT/%{lib_oozie}/webapps/oozie/docs
-%__ln_s -f %{doc_oozie} $RPM_BUILD_ROOT/%{lib_oozie}/webapps/oozie/docs
-
-# Oozie server
-%__rm -rf $RPM_BUILD_ROOT/%{lib_oozie}/lib/hadoop-*.jar
-%__ln_s -f %{lib_hadoop}/client/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-hdfs.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-app.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-common.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-core.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-jobclient.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-mapreduce-client-shuffle.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-yarn-api.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-yarn-client.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-yarn-common.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-%__ln_s -f %{lib_hadoop}/client/hadoop-yarn-server-common.jar $RPM_BUILD_ROOT/%{lib_oozie}/lib/
-
-%__install -d -m 0755 $RPM_BUILD_ROOT/usr/bin
-
-%__install -d -m 0755 %{buildroot}/%{_localstatedir}/log/oozie
-%__install -d -m 0755 %{buildroot}/%{_localstatedir}/run/oozie
-
-%pre
-getent group oozie >/dev/null || /usr/sbin/groupadd -r oozie >/dev/null
-getent passwd oozie >/dev/null || /usr/sbin/useradd --comment "Oozie User" --shell /bin/false -M -r -g oozie --home %{data_oozie} oozie >/dev/null
-
-%post
-%{alternatives_cmd} --install %{conf_oozie} %{name}-conf %{conf_oozie_dist} 30
-%{alternatives_cmd} --install %{tomcat_conf_oozie} %{name}-tomcat-conf %{tomcat_conf_oozie}.http 30
-%{alternatives_cmd} --install %{tomcat_conf_oozie} %{name}-tomcat-conf %{tomcat_conf_oozie}.https 20
-
-/sbin/chkconfig --add oozie
-
-%preun
-if [ "$1" = 0 ]; then
- rm -r /etc/oozie/conf/tomcat-conf
- /sbin/service oozie stop > /dev/null
- /sbin/chkconfig --del oozie
- %{alternatives_cmd} --remove %{name}-tomcat-conf %{tomcat_conf_oozie}.http || :
- %{alternatives_cmd} --remove %{name}-tomcat-conf %{tomcat_conf_oozie}.https || :
- %{alternatives_cmd} --remove %{name}-conf %{conf_oozie_dist} || :
-fi
-
-%postun
-if [ $1 -ge 1 ]; then
- /sbin/service oozie condrestart > /dev/null
-fi
-
-%files
-%defattr(-,root,root)
-%config(noreplace) %{conf_oozie_dist}
-%config(noreplace) %{tomcat_conf_oozie}.*
-%{usr_bin}/oozie-setup
-%{lib_oozie}/bin/oozie-sys.sh
-%{lib_oozie}/bin/oozie-env.sh
-%{lib_oozie}/bin/oozied.sh
-%{lib_oozie}/bin/ooziedb.sh
-%{lib_oozie}/bin/oozie-setup.sh
-%{lib_oozie}/webapps
-%{lib_oozie}/libtools
-%{lib_oozie}/lib
-%{lib_oozie}/oozie-sharelib.tar.gz
-%{lib_oozie}/libext
-%{lib_oozie}/tomcat-deployment.sh
-%{initd_dir}/oozie
-%defattr(-, oozie, oozie)
-%dir %{_sysconfdir}/%{name}
-%dir %{_localstatedir}/log/oozie
-%dir %{_localstatedir}/run/oozie
-%attr(0755,oozie,oozie) %{data_oozie}
-
-%files client
-%defattr(-,root,root)
-%{usr_bin}/oozie
-%dir %{lib_oozie}
-%{lib_oozie}/bin/oozie
-%{lib_oozie}/conf/oozie-client-env.sh
-%{lib_oozie}/lib
-%doc %{doc_oozie}
-%{man_dir}/man1/oozie.1.*
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-packages/src/rpm/oozie/SRPMS/.gitignore
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/oozie/SRPMS/.gitignore b/bigtop-packages/src/rpm/oozie/SRPMS/.gitignore
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-tests/smoke-tests/oozie/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/oozie/build.gradle b/bigtop-tests/smoke-tests/oozie/build.gradle
deleted file mode 100644
index 9d9f8f0..0000000
--- a/bigtop-tests/smoke-tests/oozie/build.gradle
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-def tests_to_include() {
- return [
- "TestOozieSmoke.groovy"
- ];
-}
-
-dependencies {
- if (System.env.HADOOP_CONF_DIR) testRuntime files(System.env.HADOOP_CONF_DIR)
-}
-
-sourceSets {
- test {
- groovy {
- srcDirs = [
- "${BIGTOP_HOME}/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke"
- ]
- exclude {
- FileTreeElement elem -> (doExclude(elem.getName()))
- }
- }
- }
-}
-
-test.doFirst {
- checkEnv(["HADOOP_CONF_DIR", "OOZIE_URL", "OOZIE_TAR_HOME"])
- // OOZIE_TAR_HOME should point to the directory where oozie-examples.tar.gz is located
- // Usually, it could be found in $HADOOP_HOME/share/doc/oozie
- systemProperty "org.apache.bigtop.itest.oozie_tar_home", System.getenv('OOZIE_TAR_HOME')
-}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-tests/test-artifacts/fatjar/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/fatjar/pom.xml b/bigtop-tests/test-artifacts/fatjar/pom.xml
index 961747e..28f5370 100644
--- a/bigtop-tests/test-artifacts/fatjar/pom.xml
+++ b/bigtop-tests/test-artifacts/fatjar/pom.xml
@@ -75,12 +75,6 @@
</dependency>
<dependency>
<groupId>org.apache.bigtop.itest</groupId>
- <artifactId>oozie-smoke</artifactId>
- <version>${project.version}</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.bigtop.itest</groupId>
<artifactId>solr-smoke</artifactId>
<version>${project.version}</version>
<optional>true</optional>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-tests/test-artifacts/oozie/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/oozie/pom.xml b/bigtop-tests/test-artifacts/oozie/pom.xml
deleted file mode 100644
index 22fc597..0000000
--- a/bigtop-tests/test-artifacts/oozie/pom.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.bigtop.itest</groupId>
- <artifactId>bigtop-smokes</artifactId>
- <version>1.3.0</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <groupId>org.apache.bigtop.itest</groupId>
- <artifactId>oozie-smoke</artifactId>
- <version>1.3.0</version>
- <name>ooziesmoke</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a745f030/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
deleted file mode 100644
index 657dd65..0000000
--- a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.bigtop.itest.ooziesmoke
-
-import org.junit.Test
-
-import static junit.framework.Assert.assertNotNull
-import static org.junit.Assert.assertTrue
-import org.apache.bigtop.itest.shell.Shell
-import org.junit.AfterClass
-import org.junit.BeforeClass
-import static junit.framework.Assert.assertEquals
-import org.apache.hadoop.conf.Configuration
-
-class TestOozieSmoke {
- private static final int WAIT_TIMEOUT = 60000;
- private static Shell sh = new Shell("/bin/bash -s");
- private static String tmp_dir = "oozie.${(new Date().getTime())}";
-
- private static String oozie_url;
- private static String resourcemanager;
- private static String namenode;
- private static String oozie_tar_home;
-
- @BeforeClass
- static void setUp() {
- Configuration conf = new Configuration();
- conf.addResource('yarn-site.xml');
-
- oozie_url = System.getProperty("org.apache.bigtop.itest.oozie_url", "http://localhost:11000/oozie");
- resourcemanager = conf.get("yarn.resourcemanager.address")
- resourcemanager = System.getProperty("org.apache.bigtop.itest.resourcemanager", resourcemanager);
- namenode = conf.get('fs.defaultFS') ? conf.get('fs.defaultFS') : conf.get('fs.default.name')
- namenode = System.getProperty("org.apache.bigtop.itest.namenode", namenode);
- assertNotNull("resourcemanager hostname isn't set", resourcemanager)
- assertNotNull("namenode hostname isn't set", namenode)
-
- oozie_tar_home = System.getProperty("org.apache.bigtop.itest.oozie_tar_home",
- (new File("/usr/share/doc/packages/oozie/")).exists() ?
- "/usr/share/doc/packages/oozie/" :
- "/usr/share/doc/oozie*/");
-
- sh.exec("mkdir /tmp/${tmp_dir}",
- "cd /tmp/${tmp_dir}",
- "tar xzf ${oozie_tar_home}/oozie-examples.tar.gz",
- "hadoop fs -mkdir ${tmp_dir}",
- "hadoop fs -put examples ${tmp_dir}");
- assertEquals("Failed to put examples onto HDFS",
- 0, sh.ret);
- }
-
- @AfterClass
- static void tearDown() {
- sh.exec("rm -rf /tmp/${tmp_dir}",
- "hadoop fs -rmr ${tmp_dir}");
- }
-
- void testOozieExamplesCommon(String testname) {
- sh.exec("oozie job -oozie ${oozie_url} -run -DjobTracker=${resourcemanager} -DnameNode=${namenode} " +
- "-DexamplesRoot=${tmp_dir}/examples -config /tmp/${tmp_dir}/examples/apps/${testname}/job.properties");
- assertEquals("Oozie job submition ${testname} failed",
- 0, sh.ret);
-
- String jobId = sh.out[0].replaceAll(/job: /, "");
- while (sh.exec("oozie job -oozie ${oozie_url} -info ${jobId}").out.join(' ') =~ /Status\s*:\s*RUNNING/) {
- sleep(WAIT_TIMEOUT);
- }
- assertTrue("Oozie job ${testname} returned ${sh.out.join(' ')} instead of SUCCEEDED",
- (sh.out.join(' ') =~ /Status\s*:\s*SUCCEEDED/).find());
- }
-
- @Test(timeout = 300000L)
- public void testNoOp() {
- testOozieExamplesCommon("no-op");
- }
-
- @Test(timeout = 300000L)
- public void testJavaMain() {
- testOozieExamplesCommon("java-main");
- }
-
- @Test(timeout = 300000L)
- public void testMapReduce() {
- testOozieExamplesCommon("map-reduce");
- }
-
- @Test(timeout = 300000L)
- public void testCustomMain() {
- testOozieExamplesCommon("custom-main");
- }
-
- @Test(timeout = 300000L)
- public void testHadoopEl() {
- testOozieExamplesCommon("hadoop-el");
- }
-
- @Test(timeout = 300000L)
- public void testStreaming() {
- testOozieExamplesCommon("streaming");
- }
-
- @Test(timeout = 300000L)
- public void testPig() {
- testOozieExamplesCommon("pig");
- }
-
- @Test(timeout = 300000L)
- public void testHive() {
- testOozieExamplesCommon("hive");
- }
-
- @Test(timeout = 300000L)
- public void testSubwf() {
- testOozieExamplesCommon("subwf");
- }
-
- @Test(timeout = 300000L)
- public void testSsh() {
- // testOozieExamplesCommon("ssh");
- }
-
- @Test(timeout = 300000L)
- public void testDemo() {
- // testOozieExamplesCommon("demo");
- }
-}