You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/09/23 03:03:53 UTC

[beam] tag nightly-master updated (0e90411 -> 9eaef36)

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.


*** WARNING: tag nightly-master was modified! ***

    from 0e90411  (commit)
      to 9eaef36  (commit)
    from 0e90411  Merge pull request #12880: [BEAM-10933] Adjust GBK type before creating the pipeline proto
     add 7a2a439  [BEAM-7746] Add type checking to runners.pipeline_context
     add 7d67130  Merge pull request #12885 [BEAM-7746] Add type checking to runners.pipeline_context
     add c3654f6  [BEAM-9372][BEAM-9980] Makes the Python version in Flink VR suite configurable and exercise all supported versions in postcommit execution. (#12895)
     add 97323cc   [BEAM-7372][BEAM-9980] Cleans up Flink precommit VR suite definition and makes Python version configurable. (#12898)
     add 55e50f1  [BEAM-10916] Remove experimental annotations for BQ storage API source (#12767)
     add ded8d41  [BEAM-9898] Add stub with imports to apache_beam.io.snowflake
     add dad8470  Merge pull request #12758: [BEAM-9898] Move snowflake.py from apache_beam.io to apache_beam.io.external for backwards compatibility
     add b80117e  [BEAM-10871] Fix FhirLROIT tests (#12902)
     add d2e03ed  [BEAM-2546] Add InfluxDbIO (#11459)
     add 7fec038  Upgrade GCS IO to 2.1.5 and Google OAuth to 1.31.0
     add 8f0da17  Merge pull request #12877: Upgrade GCS IO to 2.1.5 and Google OAuth to 1.31.0
     add a190ae0  [BEAM-9680] Add Aggregation Min and Max lessons to Go SDK katas (#12861)
     add 2ff8337  Merge pull request #12762 from [BEAM-10948] Ensuring that BigQuery jobs are tagged with the Dataflow step that launches them
     add 52ab870  Merge pull request #12900 from [BEAM-10941] Use standard sharding conventions for fileio writes.
     add 7c9370e  Add record_pipeline and clear to RM and fix duration limiter
     add f119319  Add comments to RecordingManager
     add 89c3f51  Merge pull request #12799 from [BEAM-10603] Add record_pipeline, clear to RM and fix duration limiter
     add e1fb49b  [BEAM-10532] convert NUMERIC field in TableSchema
     add 8ff41e2  Merge pull request #12730 from mouyang/BEAM-10532
     add bada591  [BEAM-10894] Support for more pandas formats. (#12844)
     add 370652b  Move ZetaSQL UDF tests into separate class.
     add fcc1961  Move testCreateFunctionNoSelectThrowsException into ZetaSqlUdfTest.
     add e3cdcc3  Merge pull request #12901 from ibzib/zetasql-udf
     add 97d548e  [BEAM-10768] Sleep in time-based flush test to ensure correct ordering.
     add a4ea68c  Add comment explaining flush callbacks.
     add 233ddc1  Merge pull request #12637 from ibzib/BEAM-10768
     add 93f1076  [BEAM-9547] Raise NotImplementedError and WontImplementError throughout DeferredDataframe (#12858)
     add 32b5518  [BEAM-4091] Pass type hints in ptransform_fn (#9907)
     add c4ab382  Sets sdk_harness_container_images property for all UW jobs
     add 6d377a3  Merge pull request #12853: [BEAM-3083] Sets sdk_harness_container_images property for all Dataflow Runner V2 jobs
     add 115351e  [BEAM-10716] TestPubsub/TestPubsubSignal clean up subscriptions (#12830)
     add 9eaef36  Revert "[BEAM-9547] Raise NotImplementedError and WontImplementError throughout DeferredDataframe (#12858)" (#12910)

No new revisions were added by this update.

Summary of changes:
 .github/PULL_REQUEST_TEMPLATE.md                   |   2 +-
 .test-infra/jenkins/README.md                      |   4 +-
 .../job_PerformanceTests_InfluxDBIO_IT.groovy      |  54 ++
 ...PostCommit_Python_ValidatesRunner_Flink.groovy} |  10 +-
 ...b_PreCommit_Python_ValidatesRunner_Flink.groovy |   6 +-
 .test-infra/kubernetes/influxdb/influxdb.yml       |  76 +++
 .test-infra/kubernetes/influxdb/setup.sh           |  20 +
 .test-infra/kubernetes/influxdb/teardown.sh        |  19 +
 CHANGES.md                                         |   8 +-
 .../org/apache/beam/gradle/BeamModulePlugin.groovy |   6 +-
 .../common_transforms/aggregation/lesson-info.yaml |   2 +
 .../common_transforms/aggregation/max/cmd/main.go  |  43 ++
 .../aggregation/max/pkg/task/task.go               |  25 +
 .../aggregation/{sum => max}/task-info.yaml        |   0
 .../aggregation/max/task-remote-info.yaml          |   2 +
 .../go/common_transforms/aggregation/max/task.md   |  26 +
 .../aggregation/max/test/task_test.go              |  44 ++
 .../common_transforms/aggregation/min/cmd/main.go  |  43 ++
 .../aggregation/min/pkg/task/task.go               |  25 +
 .../aggregation/{sum => min}/task-info.yaml        |   0
 .../aggregation/min/task-remote-info.yaml          |   2 +
 .../go/common_transforms/aggregation/min/task.md   |  26 +
 .../aggregation/min/test/task_test.go              |  44 ++
 learning/katas/go/course-remote-info.yaml          |   2 +-
 .../sql/zetasql/ZetaSqlDialectSpecTest.java        | 169 ------
 .../sdk/extensions/sql/zetasql/ZetaSqlUdfTest.java | 215 +++++++
 .../beam/sdk/io/gcp/bigquery/BigQueryServices.java |   4 -
 .../sdk/io/gcp/bigquery/BigQueryServicesImpl.java  |   3 -
 .../gcp/bigquery/BigQueryStorageQuerySource.java   |   3 -
 .../io/gcp/bigquery/BigQueryStorageSourceBase.java |   3 -
 .../gcp/bigquery/BigQueryStorageStreamSource.java  |   4 -
 .../gcp/bigquery/BigQueryStorageTableSource.java   |   3 -
 .../beam/sdk/io/gcp/bigquery/BigQueryUtils.java    |   2 +
 .../apache/beam/sdk/io/gcp/pubsub/TestPubsub.java  |   6 +
 .../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java   |  12 +
 .../sdk/io/gcp/bigquery/BigQueryUtilsTest.java     |  45 +-
 .../beam/sdk/io/gcp/healthcare/FhirIOLROIT.java    |  19 +-
 sdks/java/io/influxdb/build.gradle                 |  36 ++
 .../beam/sdk/io/influxdb/DBShardInformation.java   |  62 ++
 .../apache/beam/sdk/io/influxdb/InfluxDbIO.java    | 675 +++++++++++++++++++++
 .../beam/sdk/io/influxdb/ShardInformation.java     |  59 ++
 .../io/influxdb/ShardInformationByStartDate.java   |  34 +-
 .../apache/beam/sdk/io/influxdb/package-info.java  |  30 +-
 .../apache/beam/sdk/io/influxdb/GenerateData.java  |  81 +++
 .../sdk/io/influxdb/InfluxDBPipelineOptions.java   |  49 ++
 .../apache/beam/sdk/io/influxdb/InfluxDbIOIT.java  | 376 ++++++++++++
 .../beam/sdk/io/influxdb/InfluxDbIOTest.java       | 211 +++++++
 .../org/apache/beam/sdk/io/influxdb/Model.java     |  99 +++
 sdks/python/apache_beam/dataframe/frame_base.py    |   4 +-
 sdks/python/apache_beam/dataframe/frames.py        |   9 +-
 sdks/python/apache_beam/dataframe/io.py            | 167 ++++-
 sdks/python/apache_beam/dataframe/io_test.py       |  95 +++
 .../io/external/snowflake.py}                      |  24 +-
 sdks/python/apache_beam/io/fileio.py               | 110 ++--
 sdks/python/apache_beam/io/fileio_test.py          |  33 +-
 sdks/python/apache_beam/io/gcp/bigquery.py         |  62 +-
 .../apache_beam/io/gcp/bigquery_file_loads.py      |  51 +-
 .../apache_beam/io/gcp/bigquery_io_metadata.py     |  16 +-
 .../python/apache_beam/options/pipeline_options.py |  43 ++
 .../options/pipeline_options_validator_test.py     |  19 +
 .../runners/dataflow/internal/apiclient.py         |   5 +-
 .../runners/interactive/background_caching_job.py  |   1 -
 .../interactive/options/capture_limiters.py        |   9 +-
 .../interactive/options/capture_limiters_test.py   |  16 +-
 .../runners/interactive/recording_manager.py       | 113 ++--
 .../runners/interactive/recording_manager_test.py  | 144 ++++-
 .../apache_beam/runners/interactive/utils.py       |  24 +-
 .../python/apache_beam/runners/pipeline_context.py |  43 +-
 .../apache_beam/runners/worker/data_plane_test.py  |  33 +-
 sdks/python/apache_beam/transforms/ptransform.py   |  29 +-
 sdks/python/apache_beam/transforms/util.py         |   2 +-
 sdks/python/apache_beam/typehints/decorators.py    |   4 +
 .../apache_beam/typehints/typed_pipeline_test.py   |  58 ++
 .../typehints/typed_pipeline_test_py3.py           |  32 +
 sdks/python/conftest.py                            |   7 +
 sdks/python/mypy.ini                               |   3 -
 sdks/python/scripts/generate_pydoc.sh              |   1 +
 sdks/python/test-suites/gradle.properties          |   6 +-
 sdks/python/test-suites/portable/build.gradle      |  33 +
 settings.gradle                                    |   1 +
 .../documentation/io/built-in/google-bigquery.md   |  41 +-
 .../en/documentation/sdks/python-type-safety.md    |   7 +
 82 files changed, 3386 insertions(+), 548 deletions(-)
 create mode 100644 .test-infra/jenkins/job_PerformanceTests_InfluxDBIO_IT.groovy
 rename .test-infra/jenkins/{job_PostCommit_Python35_ValidatesRunner_Flink.groovy => job_PostCommit_Python_ValidatesRunner_Flink.groovy} (77%)
 create mode 100644 .test-infra/kubernetes/influxdb/influxdb.yml
 create mode 100755 .test-infra/kubernetes/influxdb/setup.sh
 create mode 100755 .test-infra/kubernetes/influxdb/teardown.sh
 create mode 100644 learning/katas/go/common_transforms/aggregation/max/cmd/main.go
 create mode 100644 learning/katas/go/common_transforms/aggregation/max/pkg/task/task.go
 copy learning/katas/go/common_transforms/aggregation/{sum => max}/task-info.yaml (100%)
 create mode 100644 learning/katas/go/common_transforms/aggregation/max/task-remote-info.yaml
 create mode 100644 learning/katas/go/common_transforms/aggregation/max/task.md
 create mode 100644 learning/katas/go/common_transforms/aggregation/max/test/task_test.go
 create mode 100644 learning/katas/go/common_transforms/aggregation/min/cmd/main.go
 create mode 100644 learning/katas/go/common_transforms/aggregation/min/pkg/task/task.go
 copy learning/katas/go/common_transforms/aggregation/{sum => min}/task-info.yaml (100%)
 create mode 100644 learning/katas/go/common_transforms/aggregation/min/task-remote-info.yaml
 create mode 100644 learning/katas/go/common_transforms/aggregation/min/task.md
 create mode 100644 learning/katas/go/common_transforms/aggregation/min/test/task_test.go
 create mode 100644 sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlUdfTest.java
 create mode 100644 sdks/java/io/influxdb/build.gradle
 create mode 100644 sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/DBShardInformation.java
 create mode 100644 sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java
 create mode 100644 sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/ShardInformation.java
 copy .test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy => sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/ShardInformationByStartDate.java (50%)
 copy .test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy => sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/package-info.java (50%)
 create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/GenerateData.java
 create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDBPipelineOptions.java
 create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOIT.java
 create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOTest.java
 create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/Model.java
 copy sdks/python/{conftest.py => apache_beam/io/external/snowflake.py} (66%)
 create mode 100644 sdks/python/test-suites/portable/build.gradle