You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2018/12/02 20:51:32 UTC
[GitHub] kaxil closed pull request #4263: [AIRFLOW-XXX] Fix display of
SageMaker operators/hook docs
kaxil closed pull request #4263: [AIRFLOW-XXX] Fix display of SageMaker operators/hook docs
URL: https://github.com/apache/incubator-airflow/pull/4263
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/contrib/hooks/sagemaker_hook.py b/airflow/contrib/hooks/sagemaker_hook.py
index 823f430db9..55e62a78e2 100644
--- a/airflow/contrib/hooks/sagemaker_hook.py
+++ b/airflow/contrib/hooks/sagemaker_hook.py
@@ -160,7 +160,7 @@ def configure_s3_resources(self, config):
:param config: config of SageMaker operation
:type config: dict
- :return: dict
+ :rtype: dict
"""
s3_operations = config.pop('S3Operations', None)
@@ -182,8 +182,8 @@ def check_s3_url(self, s3url):
Check if an S3 URL exists
:param s3url: S3 url
- :type s3url:str
- :return: bool
+ :type s3url: str
+ :rtype: bool
"""
bucket, key = S3Hook.parse_s3_url(s3url)
if not self.s3_hook.check_for_bucket(bucket_name=bucket):
@@ -226,7 +226,7 @@ def get_conn(self):
"""
Establish an AWS connection for SageMaker
- :return: a boto3 SageMaker client
+ :rtype: :py:class:`SageMaker.Client`
"""
return self.get_client_type('sagemaker')
@@ -234,7 +234,7 @@ def get_log_conn(self):
"""
Establish an AWS connection for retrieving logs during training
- :return: a boto3 CloudWatchLog client
+ :rtype: :py:class:`CloudWatchLog.Client`
"""
config = botocore.config.Config(retries={'max_attempts': 15})
return self.get_client_type('logs', config=config)
@@ -253,10 +253,11 @@ def log_stream(self, log_group, stream_name, start_time=0, skip=0):
:param skip: The number of log entries to skip at the start (default: 0).
This is for when there are multiple entries at the same timestamp.
:type skip: int
- :return:A CloudWatch log event with the following key-value pairs:
- 'timestamp' (int): The time in milliseconds of the event.
- 'message' (str): The log event data.
- 'ingestionTime' (int): The time in milliseconds the event was ingested.
+ :rtype: dict
+ :return: | A CloudWatch log event with the following key-value pairs:
+ | 'timestamp' (int): The time in milliseconds of the event.
+ | 'message' (str): The log event data.
+ | 'ingestionTime' (int): The time in milliseconds the event was ingested.
"""
next_token = None
diff --git a/airflow/contrib/operators/sagemaker_endpoint_config_operator.py b/airflow/contrib/operators/sagemaker_endpoint_config_operator.py
index a94cf30229..e4f8ce7c91 100644
--- a/airflow/contrib/operators/sagemaker_endpoint_config_operator.py
+++ b/airflow/contrib/operators/sagemaker_endpoint_config_operator.py
@@ -31,12 +31,11 @@ class SageMakerEndpointConfigOperator(SageMakerBaseOperator):
:param config: The configuration necessary to create an endpoint config.
- For details of the configuration parameter, See:
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_endpoint_config
+ For details of the configuration parameter see :py:meth:`SageMaker.Client.create_endpoint_config`
:type config: dict
:param aws_conn_id: The AWS connection ID to use.
:type aws_conn_id: str
- """ # noqa: E501
+ """
integer_fields = [
['ProductionVariants', 'InitialInstanceCount']
diff --git a/airflow/contrib/operators/sagemaker_endpoint_operator.py b/airflow/contrib/operators/sagemaker_endpoint_operator.py
index 4094fbe59e..b3f17df128 100644
--- a/airflow/contrib/operators/sagemaker_endpoint_operator.py
+++ b/airflow/contrib/operators/sagemaker_endpoint_operator.py
@@ -33,43 +33,42 @@ class SageMakerEndpointOperator(SageMakerBaseOperator):
:param config:
The configuration necessary to create an endpoint.
- If you need to create a SageMaker endpoint based on an existed SageMaker model and an existed SageMaker
- endpoint config,
+ If you need to create a SageMaker endpoint based on an existed
+ SageMaker model and an existed SageMaker endpoint config::
config = endpoint_configuration;
- If you need to create all of SageMaker model, SageMaker endpoint-config and SageMaker endpoint,
+ If you need to create all of SageMaker model, SageMaker endpoint-config and SageMaker endpoint::
config = {
'Model': model_configuration,
-
'EndpointConfig': endpoint_config_configuration,
-
'Endpoint': endpoint_configuration
}
- For details of the configuration parameter of model_configuration, See:
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_model
+ For details of the configuration parameter of model_configuration see
+ :py:meth:`SageMaker.Client.create_model`
- For details of the configuration parameter of endpoint_config_configuration, See:
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_endpoint_config
+ For details of the configuration parameter of endpoint_config_configuration see
+ :py:meth:`SageMaker.Client.create_endpoint_config`
+
+ For details of the configuration parameter of endpoint_configuration see
+ :py:meth:`SageMaker.Client.create_endpoint`
- For details of the configuration parameter of endpoint_configuration, See:
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_endpoint
:type config: dict
:param aws_conn_id: The AWS connection ID to use.
:type aws_conn_id: str
:param wait_for_completion: Whether the operator should wait until the endpoint creation finishes.
:type wait_for_completion: bool
- :param check_interval: If wait is set to True, this is the time interval, in seconds, that this operation waits
- before polling the status of the endpoint creation.
+ :param check_interval: If wait is set to True, this is the time interval, in seconds, that this operation
+ waits before polling the status of the endpoint creation.
:type check_interval: int
- :param max_ingestion_time: If wait is set to True, this operation fails if the endpoint creation doesn't finish
- within max_ingestion_time seconds. If you set this parameter to None it never times out.
+ :param max_ingestion_time: If wait is set to True, this operation fails if the endpoint creation doesn't
+ finish within max_ingestion_time seconds. If you set this parameter to None it never times out.
:type max_ingestion_time: int
:param operation: Whether to create an endpoint or update an endpoint. Must be either 'create or 'update'.
:type operation: str
- """ # noqa: E501
+ """
@apply_defaults
def __init__(self,
diff --git a/airflow/contrib/operators/sagemaker_model_operator.py b/airflow/contrib/operators/sagemaker_model_operator.py
index 4332daa9db..5bfc1feb31 100644
--- a/airflow/contrib/operators/sagemaker_model_operator.py
+++ b/airflow/contrib/operators/sagemaker_model_operator.py
@@ -32,12 +32,11 @@ class SageMakerModelOperator(SageMakerBaseOperator):
:param config: The configuration necessary to create a model.
- For details of the configuration parameter, See:
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_model
+ For details of the configuration parameter see :py:meth:`SageMaker.Client.create_model`
:type config: dict
:param aws_conn_id: The AWS connection ID to use.
:type aws_conn_id: str
- """ # noqa: E501
+ """
@apply_defaults
def __init__(self,
diff --git a/airflow/contrib/operators/sagemaker_training_operator.py b/airflow/contrib/operators/sagemaker_training_operator.py
index d90f7e6555..f10e13e973 100644
--- a/airflow/contrib/operators/sagemaker_training_operator.py
+++ b/airflow/contrib/operators/sagemaker_training_operator.py
@@ -31,8 +31,7 @@ class SageMakerTrainingOperator(SageMakerBaseOperator):
:param config: The configuration necessary to start a training job (templated).
- For details of the configuration parameter, See:
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_training_job
+ For details of the configuration parameter see :py:meth:`SageMaker.Client.create_training_job`
:type config: dict
:param aws_conn_id: The AWS connection ID to use.
:type aws_conn_id: str
@@ -48,7 +47,7 @@ class SageMakerTrainingOperator(SageMakerBaseOperator):
doesn't finish within max_ingestion_time seconds. If you set this parameter to None,
the operation does not timeout.
:type max_ingestion_time: int
- """ # noqa: E501
+ """
integer_fields = [
['ResourceConfig', 'InstanceCount'],
diff --git a/airflow/contrib/operators/sagemaker_transform_operator.py b/airflow/contrib/operators/sagemaker_transform_operator.py
index 9d1c665f9e..9dcf9ab327 100644
--- a/airflow/contrib/operators/sagemaker_transform_operator.py
+++ b/airflow/contrib/operators/sagemaker_transform_operator.py
@@ -31,23 +31,22 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
:param config: The configuration necessary to start a transform job (templated).
- If you need to create a SageMaker transform job based on an existed SageMaker model,
+ If you need to create a SageMaker transform job based on an existed SageMaker model::
- config = transform_config;
+ config = transform_config
- If you need to create both SageMaker model and SageMaker Transform job,
+ If you need to create both SageMaker model and SageMaker Transform job::
config = {
'Model': model_config,
-
'Transform': transform_config
}
- For details of the configuration parameter of transform_config, See:
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_transform_job
+ For details of the configuration parameter of transform_config see
+ :py:meth:`SageMaker.Client.create_transform_job`
For details of the configuration parameter of model_config, See:
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_model
+ :py:meth:`SageMaker.Client.create_model`
:type config: dict
:param aws_conn_id: The AWS connection ID to use.
@@ -61,7 +60,7 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
if the transform job doesn't finish within max_ingestion_time seconds. If you
set this parameter to None, the operation does not timeout.
:type max_ingestion_time: int
- """ # noqa: E501
+ """
@apply_defaults
def __init__(self,
diff --git a/airflow/contrib/operators/sagemaker_tuning_operator.py b/airflow/contrib/operators/sagemaker_tuning_operator.py
index dc1282b5b8..2a92bf1b81 100644
--- a/airflow/contrib/operators/sagemaker_tuning_operator.py
+++ b/airflow/contrib/operators/sagemaker_tuning_operator.py
@@ -31,8 +31,8 @@ class SageMakerTuningOperator(SageMakerBaseOperator):
:param config: The configuration necessary to start a tuning job (templated).
- For details of the configuration parameter, See:
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_hyper_parameter_tuning_job
+ For details of the configuration parameter see
+ :py:meth:`SageMaker.Client.create_hyper_parameter_tuning_job`
:type config: dict
:param aws_conn_id: The AWS connection ID to use.
:type aws_conn_id: str
@@ -45,7 +45,7 @@ class SageMakerTuningOperator(SageMakerBaseOperator):
if the tuning job doesn't finish within max_ingestion_time seconds. If you
set this parameter to None, the operation does not timeout.
:type max_ingestion_time: int
- """ # noqa: E501
+ """
integer_fields = [
['HyperParameterTuningJobConfig', 'ResourceLimits', 'MaxNumberOfTrainingJobs'],
diff --git a/docs/conf.py b/docs/conf.py
index cc00a27641..7c17f97bf9 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -76,7 +76,8 @@
'sphinx.ext.coverage',
'sphinx.ext.viewcode',
'sphinxarg.ext',
- 'sphinxcontrib.httpdomain'
+ 'sphinxcontrib.httpdomain',
+ 'sphinx.ext.intersphinx',
]
autodoc_default_flags = ['show-inheritance', 'members']
@@ -147,6 +148,10 @@
#keep_warnings = False
+intersphinx_mapping = {
+ 'boto3': ('https://boto3.amazonaws.com/v1/documentation/api/latest', None),
+}
+
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services