You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/02/10 16:19:27 UTC
incubator-airflow git commit: [AIRFLOW-2085] Add SparkJdbc operator
Repository: incubator-airflow
Updated Branches:
refs/heads/master 7488f2938 -> d5b5ae0ee
[AIRFLOW-2085] Add SparkJdbc operator
Add the Spark JDBC hook/operator pair. This pair
extends
the existing SparkSubmitOperator. Also includes
the
spark_jdbc_script, which is the actual Spark job
that is run.
Closes #3021 from danielvdende/AIRFLOW-2085-add-
spark-jdbc-operator
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/d5b5ae0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/d5b5ae0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/d5b5ae0e
Branch: refs/heads/master
Commit: d5b5ae0ee4089261916028390bcb3fad63c7f022
Parents: 7488f29
Author: Daniel van der Ende <da...@gmail.com>
Authored: Sat Feb 10 17:19:18 2018 +0100
Committer: Fokko Driesprong <fo...@godatadriven.com>
Committed: Sat Feb 10 17:19:18 2018 +0100
----------------------------------------------------------------------
airflow/contrib/hooks/spark_jdbc_hook.py | 241 +++++++++++++++++++
airflow/contrib/hooks/spark_jdbc_script.py | 141 +++++++++++
.../contrib/operators/spark_jdbc_operator.py | 209 ++++++++++++++++
tests/contrib/hooks/test_spark_jdbc_hook.py | 134 +++++++++++
.../operators/test_spark_jdbc_operator.py | 143 +++++++++++
5 files changed, 868 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5b5ae0e/airflow/contrib/hooks/spark_jdbc_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/spark_jdbc_hook.py b/airflow/contrib/hooks/spark_jdbc_hook.py
new file mode 100644
index 0000000..cbc35b1
--- /dev/null
+++ b/airflow/contrib/hooks/spark_jdbc_hook.py
@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook
+from airflow.exceptions import AirflowException
+
+
+class SparkJDBCHook(SparkSubmitHook):
+ """
+ This hook extends the SparkSubmitHook specifically for performing data
+ transfers to/from JDBC-based databases with Apache Spark.
+
+ :param spark_app_name: Name of the job (default airflow-spark-jdbc)
+ :type spark_app_name: str
+ :param spark_conn_id: Connection id as configured in Airflow administration
+ :type spark_conn_id: str
+ :param spark_conf: Any additional Spark configuration properties
+ :type spark_conf: dict
+ :param spark_py_files: Additional python files used (.zip, .egg, or .py)
+ :type spark_py_files: str
+ :param spark_files: Additional files to upload to the container running the job
+ :type spark_files: str
+ :param spark_jars: Additional jars to upload and add to the driver and
+ executor classpath
+ :type spark_jars: str
+ :param num_executors: number of executor to run. This should be set so as to manage
+ the number of connections made with the JDBC database
+ :type num_executors: int
+ :param executor_cores: Number of cores per executor
+ :type executor_cores: int
+ :param executor_memory: Memory per executor (e.g. 1000M, 2G)
+ :type executor_memory: str
+ :param driver_memory: Memory allocated to the driver (e.g. 1000M, 2G)
+ :type driver_memory: str
+ :param verbose: Whether to pass the verbose flag to spark-submit for debugging
+ :type verbose: bool
+ :param keytab: Full path to the file that contains the keytab
+ :type keytab: str
+ :param principal: The name of the kerberos principal used for keytab
+ :type principal: str
+ :param cmd_type: Which way the data should flow. 2 possible values:
+ spark_to_jdbc: data written by spark from metastore to jdbc
+ jdbc_to_spark: data written by spark from jdbc to metastore
+ :type cmd_type: str
+ :param jdbc_table: The name of the JDBC table
+ :type jdbc_table: str
+ :param jdbc_conn_id: Connection id used for connection to JDBC database
+ :type: jdbc_conn_id: str
+ :param jdbc_driver: Name of the JDBC driver to use for the JDBC connection. This
+ driver (usually a jar) should be passed in the 'jars' parameter
+ :type jdbc_driver: str
+ :param metastore_table: The name of the metastore table,
+ :type metastore_table: str
+ :param jdbc_truncate: (spark_to_jdbc only) Whether or not Spark should truncate or
+ drop and recreate the JDBC table. This only takes effect if
+ 'save_mode' is set to Overwrite. Also, if the schema is
+ different, Spark cannot truncate, and will drop and recreate
+ :type jdbc_truncate: bool
+ :param save_mode: The Spark save-mode to use (e.g. overwrite, append, etc.)
+ :type save_mode: str
+ :param save_format: (jdbc_to_spark-only) The Spark save-format to use (e.g. parquet)
+ :type save_format: str
+ :param batch_size: (spark_to_jdbc only) The size of the batch to insert per round
+ trip to the JDBC database. Defaults to 1000
+ :type batch_size: int
+ :param fetch_size: (jdbc_to_spark only) The size of the batch to fetch per round trip
+ from the JDBC database. Default depends on the JDBC driver
+ :type fetch_size: int
+ :param num_partitions: The maximum number of partitions that can be used by Spark
+ simultaneously, both for spark_to_jdbc and jdbc_to_spark
+ operations. This will also cap the number of JDBC connections
+ that can be opened
+ :type num_partitions: int
+ :param partition_column: (jdbc_to_spark-only) A numeric column to be used to
+ partition the metastore table by. If specified, you must
+ also specify:
+ num_partitions, lower_bound, upper_bound
+ :type partition_column: str
+ :param lower_bound: (jdbc_to_spark-only) Lower bound of the range of the numeric
+ partition column to fetch. If specified, you must also specify:
+ num_partitions, partition_column, upper_bound
+ :type lower_bound: int
+ :param upper_bound: (jdbc_to_spark-only) Upper bound of the range of the numeric
+ partition column to fetch. If specified, you must also specify:
+ num_partitions, partition_column, lower_bound
+ :type upper_bound: int
+ :param create_table_column_types: (spark_to_jdbc-only) The database column data types
+ to use instead of the defaults, when creating the
+ table. Data type information should be specified in
+ the same format as CREATE TABLE columns syntax
+ (e.g: "name CHAR(64), comments VARCHAR(1024)").
+ The specified types should be valid spark sql data
+ types.
+ """
+ def __init__(self,
+ spark_app_name='airflow-spark-jdbc',
+ spark_conn_id='spark-default',
+ spark_conf=None,
+ spark_py_files=None,
+ spark_files=None,
+ spark_jars=None,
+ num_executors=None,
+ executor_cores=None,
+ executor_memory=None,
+ driver_memory=None,
+ verbose=False,
+ principal=None,
+ keytab=None,
+ cmd_type='spark_to_jdbc',
+ jdbc_table=None,
+ jdbc_conn_id='jdbc-default',
+ jdbc_driver=None,
+ metastore_table=None,
+ jdbc_truncate=False,
+ save_mode=None,
+ save_format=None,
+ batch_size=None,
+ fetch_size=None,
+ num_partitions=None,
+ partition_column=None,
+ lower_bound=None,
+ upper_bound=None,
+ create_table_column_types=None,
+ *args,
+ **kwargs
+ ):
+ super(SparkJDBCHook, self).__init__(*args, **kwargs)
+ self._name = spark_app_name
+ self._conn_id = spark_conn_id
+ self._conf = spark_conf
+ self._py_files = spark_py_files
+ self._files = spark_files
+ self._jars = spark_jars
+ self._num_executors = num_executors
+ self._executor_cores = executor_cores
+ self._executor_memory = executor_memory
+ self._driver_memory = driver_memory
+ self._verbose = verbose
+ self._keytab = keytab
+ self._principal = principal
+ self._cmd_type = cmd_type
+ self._jdbc_table = jdbc_table
+ self._jdbc_conn_id = jdbc_conn_id
+ self._jdbc_driver = jdbc_driver
+ self._metastore_table = metastore_table
+ self._jdbc_truncate = jdbc_truncate
+ self._save_mode = save_mode
+ self._save_format = save_format
+ self._batch_size = batch_size
+ self._fetch_size = fetch_size
+ self._num_partitions = num_partitions
+ self._partition_column = partition_column
+ self._lower_bound = lower_bound
+ self._upper_bound = upper_bound
+ self._create_table_column_types = create_table_column_types
+ self._jdbc_connection = self._resolve_jdbc_connection()
+
+ def _resolve_jdbc_connection(self):
+ conn_data = {'url': '',
+ 'schema': '',
+ 'conn_prefix': '',
+ 'user': '',
+ 'password': ''
+ }
+ try:
+ conn = self.get_connection(self._jdbc_conn_id)
+ if conn.port:
+ conn_data['url'] = "{}:{}".format(conn.host, conn.port)
+ else:
+ conn_data['url'] = conn.host
+ conn_data['schema'] = conn.schema
+ conn_data['user'] = conn.login
+ conn_data['password'] = conn.password
+ extra = conn.extra_dejson
+ conn_data['conn_prefix'] = extra.get('conn_prefix', '')
+ except AirflowException:
+ self.log.debug(
+ "Could not load jdbc connection string %s, defaulting to %s",
+ self._jdbc_conn_id, ""
+ )
+ return conn_data
+
+ def _build_jdbc_application_arguments(self, jdbc_conn):
+ arguments = []
+ arguments += ["-cmdType", self._cmd_type]
+ if self._jdbc_connection['url']:
+ arguments += ['-url', "{0}{1}/{2}".format(
+ jdbc_conn['conn_prefix'], jdbc_conn['url'], jdbc_conn['schema']
+ )]
+ if self._jdbc_connection['user']:
+ arguments += ['-user', self._jdbc_connection['user']]
+ if self._jdbc_connection['password']:
+ arguments += ['-password', self._jdbc_connection['password']]
+ if self._metastore_table:
+ arguments += ['-metastoreTable', self._metastore_table]
+ if self._jdbc_table:
+ arguments += ['-jdbcTable', self._jdbc_table]
+ if self._jdbc_truncate:
+ arguments += ['-jdbcTruncate', str(self._jdbc_truncate)]
+ if self._jdbc_driver:
+ arguments += ['-jdbcDriver', self._jdbc_driver]
+ if self._batch_size:
+ arguments += ['-batchsize', str(self._batch_size)]
+ if self._fetch_size:
+ arguments += ['-fetchsize', str(self._fetch_size)]
+ if self._num_partitions:
+ arguments += ['-numPartitions', str(self._num_partitions)]
+ if (self._partition_column and self._lower_bound and
+ self._upper_bound and self._num_partitions):
+ # these 3 parameters need to be used all together to take effect.
+ arguments += ['-partitionColumn', self._partition_column,
+ '-lowerBound', self._lower_bound,
+ '-upperBound', self._upper_bound]
+ if self._save_mode:
+ arguments += ['-saveMode', self._save_mode]
+ if self._save_format:
+ arguments += ['-saveFormat', self._save_format]
+ if self._create_table_column_types:
+ arguments += ['-createTableColumnTypes', self._create_table_column_types]
+ return arguments
+
+ def submit_jdbc_job(self):
+ self._application_args = \
+ self._build_jdbc_application_arguments(self._jdbc_connection)
+ self.submit(application=os.path.dirname(os.path.abspath(__file__)) +
+ "/spark_jdbc_script.py")
+
+ def get_conn(self):
+ pass
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5b5ae0e/airflow/contrib/hooks/spark_jdbc_script.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/spark_jdbc_script.py b/airflow/contrib/hooks/spark_jdbc_script.py
new file mode 100644
index 0000000..41d32aa
--- /dev/null
+++ b/airflow/contrib/hooks/spark_jdbc_script.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import argparse
+from pyspark.sql import SparkSession
+
+
+def set_common_options(spark_source,
+ url='localhost:5432',
+ jdbc_table='default.default',
+ user='root',
+ password='root',
+ driver='driver'):
+
+ spark_source = spark_source \
+ .format('jdbc') \
+ .option('url', url) \
+ .option('dbtable', jdbc_table) \
+ .option('user', user) \
+ .option('password', password) \
+ .option('driver', driver)
+ return spark_source
+
+
+def spark_write_to_jdbc(spark, url, user, password, metastore_table, jdbc_table, driver,
+ truncate, save_mode, batch_size, num_partitions,
+ create_table_column_types):
+ writer = spark \
+ .table(metastore_table) \
+ .write \
+
+ # first set common options
+ writer = set_common_options(writer, url, jdbc_table, user, password, driver)
+
+ # now set write-specific options
+ if truncate:
+ writer = writer.option('truncate', truncate)
+ if batch_size:
+ writer = writer.option('batchsize', batch_size)
+ if num_partitions:
+ writer = writer.option('numPartitions', num_partitions)
+ if create_table_column_types:
+ writer = writer.option("createTableColumnTypes", create_table_column_types)
+
+ writer \
+ .save(mode=save_mode)
+
+
+def spark_read_from_jdbc(spark, url, user, password, metastore_table, jdbc_table, driver,
+ save_mode, save_format, fetch_size, num_partitions,
+ partition_column, lower_bound, upper_bound):
+
+ # first set common options
+ reader = set_common_options(spark.read, url, jdbc_table, user, password, driver)
+
+ # now set specific read options
+ if fetch_size:
+ reader = reader.option('fetchsize', fetch_size)
+ if num_partitions:
+ reader = reader.option('numPartitions', num_partitions)
+ if partition_column and lower_bound and upper_bound:
+ reader = reader \
+ .option('partitionColumn', partition_column) \
+ .option('lowerBound', lower_bound) \
+ .option('upperBound', upper_bound)
+
+ reader \
+ .load() \
+ .write \
+ .saveAsTable(metastore_table, format=save_format, mode=save_mode)
+
+
+if __name__ == "__main__": # pragma: no cover
+ # parse the parameters
+ parser = argparse.ArgumentParser(description='Spark-JDBC')
+ parser.add_argument('-cmdType', dest='cmd_type', action='store')
+ parser.add_argument('-url', dest='url', action='store')
+ parser.add_argument('-user', dest='user', action='store')
+ parser.add_argument('-password', dest='password', action='store')
+ parser.add_argument('-metastoreTable', dest='metastore_table', action='store')
+ parser.add_argument('-jdbcTable', dest='jdbc_table', action='store')
+ parser.add_argument('-jdbcDriver', dest='jdbc_driver', action='store')
+ parser.add_argument('-jdbcTruncate', dest='truncate', action='store')
+ parser.add_argument('-saveMode', dest='save_mode', action='store')
+ parser.add_argument('-saveFormat', dest='save_format', action='store')
+ parser.add_argument('-batchsize', dest='batch_size', action='store')
+ parser.add_argument('-fetchsize', dest='fetch_size', action='store')
+ parser.add_argument('-name', dest='name', action='store')
+ parser.add_argument('-numPartitions', dest='num_partitions', action='store')
+ parser.add_argument('-partitionColumn', dest='partition_column', action='store')
+ parser.add_argument('-lowerBound', dest='lower_bound', action='store')
+ parser.add_argument('-upperBound', dest='upper_bound', action='store')
+ parser.add_argument('-createTableColumnTypes',
+ dest='create_table_column_types', action='store')
+ arguments = parser.parse_args()
+
+ # Disable dynamic allocation by default to allow num_executors to take effect.
+ spark = SparkSession.builder \
+ .appName(arguments.name) \
+ .enableHiveSupport() \
+ .getOrCreate()
+
+ if arguments.cmd_type == "spark_to_jdbc":
+ spark_write_to_jdbc(spark,
+ arguments.url,
+ arguments.user,
+ arguments.password,
+ arguments.metastore_table,
+ arguments.jdbc_table,
+ arguments.jdbc_driver,
+ arguments.truncate,
+ arguments.save_mode,
+ arguments.batch_size,
+ arguments.num_partitions,
+ arguments.create_table_column_types)
+ elif arguments.cmd_type == "jdbc_to_spark":
+ spark_read_from_jdbc(spark,
+ arguments.url,
+ arguments.user,
+ arguments.password,
+ arguments.metastore_table,
+ arguments.jdbc_table,
+ arguments.jdbc_driver,
+ arguments.save_mode,
+ arguments.save_format,
+ arguments.fetch_size,
+ arguments.num_partitions,
+ arguments.partition_column,
+ arguments.lower_bound,
+ arguments.upper_bound)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5b5ae0e/airflow/contrib/operators/spark_jdbc_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/spark_jdbc_operator.py b/airflow/contrib/operators/spark_jdbc_operator.py
new file mode 100644
index 0000000..cab4336
--- /dev/null
+++ b/airflow/contrib/operators/spark_jdbc_operator.py
@@ -0,0 +1,209 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from airflow.contrib.operators.spark_submit_operator import SparkSubmitOperator
+from airflow.contrib.hooks.spark_jdbc_hook import SparkJDBCHook
+from airflow.utils.decorators import apply_defaults
+
+
+class SparkJDBCOperator(SparkSubmitOperator):
+ """
+ This operator extends the SparkSubmitOperator specifically for performing data
+ transfers to/from JDBC-based databases with Apache Spark. As with the
+ SparkSubmitOperator, it assumes that the "spark-submit" binary is available on the
+ PATH.
+
+ :param spark_app_name: Name of the job (default airflow-spark-jdbc)
+ :type spark_app_name: str
+ :param spark_conn_id: Connection id as configured in Airflow administration
+ :type spark_conn_id: str
+ :param spark_conf: Any additional Spark configuration properties
+ :type spark_conf: dict
+ :param spark_py_files: Additional python files used (.zip, .egg, or .py)
+ :type spark_py_files: str
+ :param spark_files: Additional files to upload to the container running the job
+ :type spark_files: str
+ :param spark_jars: Additional jars to upload and add to the driver and
+ executor classpath
+ :type spark_jars: str
+ :param num_executors: number of executor to run. This should be set so as to manage
+ the number of connections made with the JDBC database
+ :type num_executors: int
+ :param executor_cores: Number of cores per executor
+ :type executor_cores: int
+ :param executor_memory: Memory per executor (e.g. 1000M, 2G)
+ :type executor_memory: str
+ :param driver_memory: Memory allocated to the driver (e.g. 1000M, 2G)
+ :type driver_memory: str
+ :param verbose: Whether to pass the verbose flag to spark-submit for debugging
+ :type verbose: bool
+ :param keytab: Full path to the file that contains the keytab
+ :type keytab: str
+ :param principal: The name of the kerberos principal used for keytab
+ :type principal: str
+ :param cmd_type: Which way the data should flow. 2 possible values:
+ spark_to_jdbc: data written by spark from metastore to jdbc
+ jdbc_to_spark: data written by spark from jdbc to metastore
+ :type cmd_type: str
+ :param jdbc_table: The name of the JDBC table
+ :type jdbc_table: str
+ :param jdbc_conn_id: Connection id used for connection to JDBC database
+ :type: jdbc_conn_id: str
+ :param jdbc_driver: Name of the JDBC driver to use for the JDBC connection. This
+ driver (usually a jar) should be passed in the 'jars' parameter
+ :type jdbc_driver: str
+ :param metastore_table: The name of the metastore table,
+ :type metastore_table: str
+ :param jdbc_truncate: (spark_to_jdbc only) Whether or not Spark should truncate or
+ drop and recreate the JDBC table. This only takes effect if
+ 'save_mode' is set to Overwrite. Also, if the schema is
+ different, Spark cannot truncate, and will drop and recreate
+ :type jdbc_truncate: bool
+ :param save_mode: The Spark save-mode to use (e.g. overwrite, append, etc.)
+ :type save_mode: str
+ :param save_format: (jdbc_to_spark-only) The Spark save-format to use (e.g. parquet)
+ :type save_format: str
+ :param batch_size: (spark_to_jdbc only) The size of the batch to insert per round
+ trip to the JDBC database. Defaults to 1000
+ :type batch_size: int
+ :param fetch_size: (jdbc_to_spark only) The size of the batch to fetch per round trip
+ from the JDBC database. Default depends on the JDBC driver
+ :type fetch_size: int
+ :param num_partitions: The maximum number of partitions that can be used by Spark
+ simultaneously, both for spark_to_jdbc and jdbc_to_spark
+ operations. This will also cap the number of JDBC connections
+ that can be opened
+ :type num_partitions: int
+ :param partition_column: (jdbc_to_spark-only) A numeric column to be used to
+ partition the metastore table by. If specified, you must
+ also specify:
+ num_partitions, lower_bound, upper_bound
+ :type partition_column: str
+ :param lower_bound: (jdbc_to_spark-only) Lower bound of the range of the numeric
+ partition column to fetch. If specified, you must also specify:
+ num_partitions, partition_column, upper_bound
+ :type lower_bound: int
+ :param upper_bound: (jdbc_to_spark-only) Upper bound of the range of the numeric
+ partition column to fetch. If specified, you must also specify:
+ num_partitions, partition_column, lower_bound
+ :type upper_bound: int
+ :param create_table_column_types: (spark_to_jdbc-only) The database column data types
+ to use instead of the defaults, when creating the
+ table. Data type information should be specified in
+ the same format as CREATE TABLE columns syntax
+ (e.g: "name CHAR(64), comments VARCHAR(1024)").
+ The specified types should be valid spark sql data
+ types.
+ """
+
+ @apply_defaults
+ def __init__(self,
+ spark_app_name='airflow-spark-jdbc',
+ spark_conn_id='spark-default',
+ spark_conf=None,
+ spark_py_files=None,
+ spark_files=None,
+ spark_jars=None,
+ num_executors=None,
+ executor_cores=None,
+ executor_memory=None,
+ driver_memory=None,
+ verbose=False,
+ keytab=None,
+ principal=None,
+ cmd_type='spark_to_jdbc',
+ jdbc_table=None,
+ jdbc_conn_id='jdbc-default',
+ jdbc_driver=None,
+ metastore_table=None,
+ jdbc_truncate=False,
+ save_mode=None,
+ save_format=None,
+ batch_size=None,
+ fetch_size=None,
+ num_partitions=None,
+ partition_column=None,
+ lower_bound=None,
+ upper_bound=None,
+ create_table_column_types=None,
+ *args,
+ **kwargs):
+ super(SparkJDBCOperator, self).__init__(*args, **kwargs)
+ self._spark_app_name = spark_app_name
+ self._spark_conn_id = spark_conn_id
+ self._spark_conf = spark_conf
+ self._spark_py_files = spark_py_files
+ self._spark_files = spark_files
+ self._spark_jars = spark_jars
+ self._num_executors = num_executors
+ self._executor_cores = executor_cores
+ self._executor_memory = executor_memory
+ self._driver_memory = driver_memory
+ self._verbose = verbose
+ self._keytab = keytab
+ self._principal = principal
+ self._cmd_type = cmd_type
+ self._jdbc_table = jdbc_table
+ self._jdbc_conn_id = jdbc_conn_id
+ self._jdbc_driver = jdbc_driver
+ self._metastore_table = metastore_table
+ self._jdbc_truncate = jdbc_truncate
+ self._save_mode = save_mode
+ self._save_format = save_format
+ self._batch_size = batch_size
+ self._fetch_size = fetch_size
+ self._num_partitions = num_partitions
+ self._partition_column = partition_column
+ self._lower_bound = lower_bound
+ self._upper_bound = upper_bound
+ self._create_table_column_types = create_table_column_types
+
+ def execute(self, context):
+ """
+ Call the SparkSubmitHook to run the provided spark job
+ """
+ self._hook = SparkJDBCHook(
+ spark_app_name=self._spark_app_name,
+ spark_conn_id=self._spark_conn_id,
+ spark_conf=self._spark_conf,
+ spark_py_files=self._spark_py_files,
+ spark_files=self._spark_files,
+ spark_jars=self._spark_jars,
+ num_executors=self._num_executors,
+ executor_cores=self._executor_cores,
+ executor_memory=self._executor_memory,
+ driver_memory=self._driver_memory,
+ verbose=self._verbose,
+ keytab=self._keytab,
+ principal=self._principal,
+ cmd_type=self._cmd_type,
+ jdbc_table=self._jdbc_table,
+ jdbc_conn_id=self._jdbc_conn_id,
+ jdbc_driver=self._jdbc_driver,
+ metastore_table=self._metastore_table,
+ jdbc_truncate=self._jdbc_truncate,
+ save_mode=self._save_mode,
+ save_format=self._save_format,
+ batch_size=self._batch_size,
+ fetch_size=self._fetch_size,
+ num_partitions=self._num_partitions,
+ partition_column=self._partition_column,
+ lower_bound=self._lower_bound,
+ upper_bound=self._upper_bound,
+ create_table_column_types=self._create_table_column_types
+ )
+ self._hook.submit_jdbc_job()
+
+ def on_kill(self):
+ self._hook.on_kill()
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5b5ae0e/tests/contrib/hooks/test_spark_jdbc_hook.py
----------------------------------------------------------------------
diff --git a/tests/contrib/hooks/test_spark_jdbc_hook.py b/tests/contrib/hooks/test_spark_jdbc_hook.py
new file mode 100644
index 0000000..377183d
--- /dev/null
+++ b/tests/contrib/hooks/test_spark_jdbc_hook.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import unittest
+
+from airflow import configuration, models
+from airflow.utils import db
+
+from airflow.contrib.hooks.spark_jdbc_hook import SparkJDBCHook
+
+
+class TestSparkJDBCHook(unittest.TestCase):
+
+ _config = {
+ 'cmd_type': 'spark_to_jdbc',
+ 'jdbc_table': 'tableMcTableFace',
+ 'jdbc_driver': 'org.postgresql.Driver',
+ 'metastore_table': 'hiveMcHiveFace',
+ 'jdbc_truncate': False,
+ 'save_mode': 'append',
+ 'save_format': 'parquet',
+ 'batch_size': 100,
+ 'fetch_size': 200,
+ 'num_partitions': 10,
+ 'partition_column': 'columnMcColumnFace',
+ 'lower_bound': '10',
+ 'upper_bound': '20',
+ 'create_table_column_types': 'columnMcColumnFace INTEGER(100), name CHAR(64),'
+ 'comments VARCHAR(1024)'
+ }
+
+ # this config is invalid because if one of [partitionColumn, lowerBound, upperBound]
+ # is set, all of the options must be enabled (enforced by Spark)
+ _invalid_config = {
+ 'cmd_type': 'spark_to_jdbc',
+ 'jdbc_table': 'tableMcTableFace',
+ 'jdbc_driver': 'org.postgresql.Driver',
+ 'metastore_table': 'hiveMcHiveFace',
+ 'jdbc_truncate': False,
+ 'save_mode': 'append',
+ 'save_format': 'parquet',
+ 'batch_size': 100,
+ 'fetch_size': 200,
+ 'num_partitions': 10,
+ 'partition_column': 'columnMcColumnFace',
+ 'upper_bound': '20',
+ 'create_table_column_types': 'columnMcColumnFace INTEGER(100), name CHAR(64),'
+ 'comments VARCHAR(1024)'
+ }
+
+ def setUp(self):
+ configuration.load_test_config()
+ db.merge_conn(
+ models.Connection(
+ conn_id='spark-default', conn_type='spark',
+ host='yarn://yarn-master',
+ extra='{"queue": "root.etl", "deploy-mode": "cluster"}')
+ )
+ db.merge_conn(
+ models.Connection(
+ conn_id='jdbc-default', conn_type='postgres',
+ host='localhost', schema='default', port=5432,
+ login='user', password='supersecret',
+ extra='{"conn_prefix":"jdbc:postgresql://"}'
+ )
+ )
+
+ def test_resolve_jdbc_connection(self):
+ # Given
+ hook = SparkJDBCHook(jdbc_conn_id='jdbc-default')
+ expected_connection = {
+ 'url': 'localhost:5432',
+ 'schema': 'default',
+ 'conn_prefix': 'jdbc:postgresql://',
+ 'user': 'user',
+ 'password': 'supersecret'
+ }
+
+ # When
+ connection = hook._resolve_jdbc_connection()
+
+ # Then
+ self.assertEqual(connection, expected_connection)
+
+ def test_build_jdbc_arguments(self):
+ # Given
+ hook = SparkJDBCHook(**self._config)
+
+ # When
+ cmd = hook._build_jdbc_application_arguments(hook._resolve_jdbc_connection())
+
+ # Then
+ expected_jdbc_arguments = [
+ '-cmdType', 'spark_to_jdbc',
+ '-url', 'jdbc:postgresql://localhost:5432/default',
+ '-user', 'user',
+ '-password', 'supersecret',
+ '-metastoreTable', 'hiveMcHiveFace',
+ '-jdbcTable', 'tableMcTableFace',
+ '-jdbcDriver', 'org.postgresql.Driver',
+ '-batchsize', '100',
+ '-fetchsize', '200',
+ '-numPartitions', '10',
+ '-partitionColumn', 'columnMcColumnFace',
+ '-lowerBound', '10',
+ '-upperBound', '20',
+ '-saveMode', 'append',
+ '-saveFormat', 'parquet',
+ '-createTableColumnTypes', 'columnMcColumnFace INTEGER(100), name CHAR(64),'
+ 'comments VARCHAR(1024)'
+ ]
+ self.assertEquals(expected_jdbc_arguments, cmd)
+
+ def test_build_jdbc_arguments_invalid(self):
+ # Given
+ hook = SparkJDBCHook(**self._invalid_config)
+
+ # Expect Exception
+ hook._build_jdbc_application_arguments(hook._resolve_jdbc_connection())
+
+
+if __name__ == '__main__':
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5b5ae0e/tests/contrib/operators/test_spark_jdbc_operator.py
----------------------------------------------------------------------
diff --git a/tests/contrib/operators/test_spark_jdbc_operator.py b/tests/contrib/operators/test_spark_jdbc_operator.py
new file mode 100644
index 0000000..5996db0
--- /dev/null
+++ b/tests/contrib/operators/test_spark_jdbc_operator.py
@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+
+from airflow import DAG, configuration
+
+from airflow.contrib.operators.spark_jdbc_operator import SparkJDBCOperator
+from airflow.utils import timezone
+
+DEFAULT_DATE = timezone.datetime(2017, 1, 1)
+
+
+class TestSparkJDBCOperator(unittest.TestCase):
+ _config = {
+ 'spark_app_name': '{{ task_instance.task_id }}',
+ 'spark_conf': {
+ 'parquet.compression': 'SNAPPY'
+ },
+ 'spark_files': 'hive-site.xml',
+ 'spark_py_files': 'sample_library.py',
+ 'spark_jars': 'parquet.jar',
+ 'num_executors': 4,
+ 'executor_cores': 4,
+ 'executor_memory': '22g',
+ 'driver_memory': '3g',
+ 'verbose': True,
+ 'keytab': 'privileged_user.keytab',
+ 'principal': 'user/spark@airflow.org',
+ 'cmd_type': 'spark_to_jdbc',
+ 'jdbc_table': 'tableMcTableFace',
+ 'jdbc_driver': 'org.postgresql.Driver',
+ 'metastore_table': 'hiveMcHiveFace',
+ 'jdbc_truncate': False,
+ 'save_mode': 'append',
+ 'save_format': 'parquet',
+ 'batch_size': 100,
+ 'fetch_size': 200,
+ 'num_partitions': 10,
+ 'partition_column': 'columnMcColumnFace',
+ 'lower_bound': '10',
+ 'upper_bound': '20',
+ 'create_table_column_types': 'columnMcColumnFace INTEGER(100), name CHAR(64),'
+ 'comments VARCHAR(1024)'
+ }
+
+ def setUp(self):
+ configuration.load_test_config()
+ args = {
+ 'owner': 'airflow',
+ 'start_date': DEFAULT_DATE
+ }
+ self.dag = DAG('test_dag_id', default_args=args)
+
+ def test_execute(self):
+ # Given / When
+ spark_conn_id = 'spark-default'
+ jdbc_conn_id = 'jdbc-default'
+
+ operator = SparkJDBCOperator(
+ task_id='spark_jdbc_job',
+ dag=self.dag,
+ **self._config
+ )
+
+ # Then
+ expected_dict = {
+ 'spark_app_name': '{{ task_instance.task_id }}',
+ 'spark_conf': {
+ 'parquet.compression': 'SNAPPY'
+ },
+ 'spark_files': 'hive-site.xml',
+ 'spark_py_files': 'sample_library.py',
+ 'spark_jars': 'parquet.jar',
+ 'num_executors': 4,
+ 'executor_cores': 4,
+ 'executor_memory': '22g',
+ 'driver_memory': '3g',
+ 'verbose': True,
+ 'keytab': 'privileged_user.keytab',
+ 'principal': 'user/spark@airflow.org',
+ 'cmd_type': 'spark_to_jdbc',
+ 'jdbc_table': 'tableMcTableFace',
+ 'jdbc_driver': 'org.postgresql.Driver',
+ 'metastore_table': 'hiveMcHiveFace',
+ 'jdbc_truncate': False,
+ 'save_mode': 'append',
+ 'save_format': 'parquet',
+ 'batch_size': 100,
+ 'fetch_size': 200,
+ 'num_partitions': 10,
+ 'partition_column': 'columnMcColumnFace',
+ 'lower_bound': '10',
+ 'upper_bound': '20',
+ 'create_table_column_types': 'columnMcColumnFace INTEGER(100), name CHAR(64),'
+ 'comments VARCHAR(1024)'
+ }
+
+ self.assertEqual(spark_conn_id, operator._spark_conn_id)
+ self.assertEqual(jdbc_conn_id, operator._jdbc_conn_id)
+ self.assertEqual(expected_dict['spark_app_name'], operator._spark_app_name)
+ self.assertEqual(expected_dict['spark_conf'], operator._spark_conf)
+ self.assertEqual(expected_dict['spark_files'], operator._spark_files)
+ self.assertEqual(expected_dict['spark_py_files'], operator._spark_py_files)
+ self.assertEqual(expected_dict['spark_jars'], operator._spark_jars)
+ self.assertEqual(expected_dict['num_executors'], operator._num_executors)
+ self.assertEqual(expected_dict['executor_cores'], operator._executor_cores)
+ self.assertEqual(expected_dict['executor_memory'], operator._executor_memory)
+ self.assertEqual(expected_dict['driver_memory'], operator._driver_memory)
+ self.assertEqual(expected_dict['verbose'], operator._verbose)
+ self.assertEqual(expected_dict['keytab'], operator._keytab)
+ self.assertEqual(expected_dict['principal'], operator._principal)
+ self.assertEqual(expected_dict['cmd_type'], operator._cmd_type)
+ self.assertEqual(expected_dict['jdbc_table'], operator._jdbc_table)
+ self.assertEqual(expected_dict['jdbc_driver'], operator._jdbc_driver)
+ self.assertEqual(expected_dict['metastore_table'], operator._metastore_table)
+ self.assertEqual(expected_dict['jdbc_truncate'], operator._jdbc_truncate)
+ self.assertEqual(expected_dict['save_mode'], operator._save_mode)
+ self.assertEqual(expected_dict['save_format'], operator._save_format)
+ self.assertEqual(expected_dict['batch_size'], operator._batch_size)
+ self.assertEqual(expected_dict['fetch_size'], operator._fetch_size)
+ self.assertEqual(expected_dict['num_partitions'], operator._num_partitions)
+ self.assertEqual(expected_dict['partition_column'], operator._partition_column)
+ self.assertEqual(expected_dict['lower_bound'], operator._lower_bound)
+ self.assertEqual(expected_dict['upper_bound'], operator._upper_bound)
+ self.assertEqual(expected_dict['create_table_column_types'],
+ operator._create_table_column_types)
+
+
+if __name__ == '__main__':
+ unittest.main()