You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by do...@apache.org on 2017/04/24 17:33:59 UTC
[14/50] [abbrv] incubator-predictionio git commit: [PIO-49]
Additional improvement to ES5 support
[PIO-49] Additional improvement to ES5 support
* Add option to build ES1 or ES5-default distribution
* Aggregate all storage subprojects
* Update deprecated SBT code
* Update Docker Compose to use official Elastic images
* Update Travis to use new build option
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/31c4bd19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/31c4bd19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/31c4bd19
Branch: refs/heads/master
Commit: 31c4bd1927eb9ae753f82b39b71ff3df7ca779de
Parents: d78b3cb
Author: Donald Szeto <do...@apache.org>
Authored: Tue Mar 7 23:16:33 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 7 23:16:33 2017 -0800
----------------------------------------------------------------------
.travis.yml | 4 +--
build.sbt | 6 +++++
make-distribution.sh | 47 ++++++++++++++++++++++++++++++++++-
storage/elasticsearch/build.sbt | 3 +--
storage/elasticsearch1/build.sbt | 2 +-
storage/hbase/build.sbt | 2 +-
storage/hdfs/build.sbt | 2 +-
storage/jdbc/build.sbt | 2 +-
storage/localfs/build.sbt | 2 +-
tests/build-docker.sh | 8 ++++--
tests/docker-compose.yml | 4 ++-
11 files changed, 69 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index b763e2d..799ec3d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,8 +46,8 @@ env:
matrix:
- BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
- - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS ES_VERSION=5
+ - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS ES_VERSION=5
before_install:
- unset SBT_OPTS JVM_OPTS
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 98444b9..fa5ba29 100644
--- a/build.sbt
+++ b/build.sbt
@@ -166,6 +166,12 @@ val root = (project in file(".")).
common,
core,
data,
+ dataElasticsearch1,
+ dataElasticsearch,
+ dataHbase,
+ dataHdfs,
+ dataJdbc,
+ dataLocalfs,
tools,
e2)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index 31954c0..c360c0e 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -19,6 +19,45 @@
set -e
+usage ()
+{
+ echo "Usage: $0 [-h|--help] [--with-es=x]"
+ echo ""
+ echo " -h|--help Show usage"
+ echo ""
+ echo " --with-es=1 Build distribution with Elasticsearch 1 support as default"
+ echo " --with-es=5 Build distribution with Elasticsearch 5 support as default"
+}
+
+ES_VERSION=1
+
+for i in "$@"
+do
+case $i in
+ -h|--help)
+ usage
+ shift
+ exit
+ ;;
+ --with-es=*)
+ ES_VERSION="${i#*=}"
+ shift
+ ;;
+ *)
+ usage
+ exit 1
+ ;;
+esac
+done
+
+if [ "$ES_VERSION" = "1" ] || [ "$ES_VERSION" = "5" ]
+then
+ echo -e "\033[0;32mBuilding with Elasticsearch $ES_VERSION support as the default choice\033[0m"
+else
+ usage
+ exit 1
+fi
+
FWDIR="$(cd `dirname $0`; pwd)"
DISTDIR="${FWDIR}/dist"
@@ -45,7 +84,13 @@ cp ${FWDIR}/project/build.properties ${DISTDIR}/project
cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt
cp ${FWDIR}/assembly/*assembly*jar ${DISTDIR}/lib
cp ${FWDIR}/assembly/spark/*jar ${DISTDIR}/lib/spark
-cp ${FWDIR}/assembly/extra/*jar ${DISTDIR}/lib/extra
+
+if [ "$ES_VERSION" = "5" ]
+then
+ mv ${DISTDIR}/lib/spark/pio-data-elasticsearch1-assembly-*.jar ${DISTDIR}/lib/extra
+else
+ mv ${DISTDIR}/lib/spark/pio-data-elasticsearch-assembly-*.jar ${DISTDIR}/lib/extra
+fi
rm -f ${DISTDIR}/lib/*javadoc.jar
rm -f ${DISTDIR}/lib/*sources.jar
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index b22cbd8..925b66f 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -53,5 +53,4 @@ assemblyShadeRules in assembly := Seq(
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")
-
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/elasticsearch1/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt
index 8c29b84..457aba2 100644
--- a/storage/elasticsearch1/build.sbt
+++ b/storage/elasticsearch1/build.sbt
@@ -43,5 +43,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "extra" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/hbase/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt
index 5856a5e..673697b 100644
--- a/storage/hbase/build.sbt
+++ b/storage/hbase/build.sbt
@@ -52,5 +52,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
index 9f064c6..35c9153 100644
--- a/storage/hdfs/build.sbt
+++ b/storage/hdfs/build.sbt
@@ -40,5 +40,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/jdbc/build.sbt
----------------------------------------------------------------------
diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt
index 63d420b..0a07ee0 100644
--- a/storage/jdbc/build.sbt
+++ b/storage/jdbc/build.sbt
@@ -43,5 +43,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/localfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt
index 2cf9977..f94b355 100644
--- a/storage/localfs/build.sbt
+++ b/storage/localfs/build.sbt
@@ -40,5 +40,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/tests/build-docker.sh
----------------------------------------------------------------------
diff --git a/tests/build-docker.sh b/tests/build-docker.sh
index 459b929..a899a18 100755
--- a/tests/build-docker.sh
+++ b/tests/build-docker.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/usr/bin/env bash -ex
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -29,7 +29,11 @@ fi
docker pull predictionio/pio-testing-base
pushd $DIR/..
-./make-distribution.sh
+if [ -z "$ES_VERSION" ]; then
+ ./make-distribution.sh
+else
+ ./make-distribution.sh --with-es=$ES_VERSION
+fi
sbt/sbt clean
mkdir assembly
cp dist/lib/*.jar assembly/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index b556f7b..b6774d3 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -16,7 +16,9 @@
version: "2"
services:
elasticsearch:
- image: elasticsearch:5-alpine
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
+ environment:
+ - xpack.security.enabled=false
hbase:
image: harisekhon/hbase:1.0
postgres: