You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/09/23 03:03:53 UTC
[beam] tag nightly-master updated (0e90411 -> 9eaef36)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 0e90411 (commit)
to 9eaef36 (commit)
from 0e90411 Merge pull request #12880: [BEAM-10933] Adjust GBK type before creating the pipeline proto
add 7a2a439 [BEAM-7746] Add type checking to runners.pipeline_context
add 7d67130 Merge pull request #12885 [BEAM-7746] Add type checking to runners.pipeline_context
add c3654f6 [BEAM-9372][BEAM-9980] Makes the Python version in Flink VR suite configurable and exercise all supported versions in postcommit execution. (#12895)
add 97323cc [BEAM-7372][BEAM-9980] Cleans up Flink precommit VR suite definition and makes Python version configurable. (#12898)
add 55e50f1 [BEAM-10916] Remove experimental annotations for BQ storage API source (#12767)
add ded8d41 [BEAM-9898] Add stub with imports to apache_beam.io.snowflake
add dad8470 Merge pull request #12758: [BEAM-9898] Move snowflake.py from apache_beam.io to apache_beam.io.external for backwards compatibility
add b80117e [BEAM-10871] Fix FhirLROIT tests (#12902)
add d2e03ed [BEAM-2546] Add InfluxDbIO (#11459)
add 7fec038 Upgrade GCS IO to 2.1.5 and Google OAuth to 1.31.0
add 8f0da17 Merge pull request #12877: Upgrade GCS IO to 2.1.5 and Google OAuth to 1.31.0
add a190ae0 [BEAM-9680] Add Aggregation Min and Max lessons to Go SDK katas (#12861)
add 2ff8337 Merge pull request #12762 from [BEAM-10948] Ensuring that BigQuery jobs are tagged with the Dataflow step that launches them
add 52ab870 Merge pull request #12900 from [BEAM-10941] Use standard sharding conventions for fileio writes.
add 7c9370e Add record_pipeline and clear to RM and fix duration limiter
add f119319 Add comments to RecordingManager
add 89c3f51 Merge pull request #12799 from [BEAM-10603] Add record_pipeline, clear to RM and fix duration limiter
add e1fb49b [BEAM-10532] convert NUMERIC field in TableSchema
add 8ff41e2 Merge pull request #12730 from mouyang/BEAM-10532
add bada591 [BEAM-10894] Support for more pandas formats. (#12844)
add 370652b Move ZetaSQL UDF tests into separate class.
add fcc1961 Move testCreateFunctionNoSelectThrowsException into ZetaSqlUdfTest.
add e3cdcc3 Merge pull request #12901 from ibzib/zetasql-udf
add 97d548e [BEAM-10768] Sleep in time-based flush test to ensure correct ordering.
add a4ea68c Add comment explaining flush callbacks.
add 233ddc1 Merge pull request #12637 from ibzib/BEAM-10768
add 93f1076 [BEAM-9547] Raise NotImplementedError and WontImplementError throughout DeferredDataframe (#12858)
add 32b5518 [BEAM-4091] Pass type hints in ptransform_fn (#9907)
add c4ab382 Sets sdk_harness_container_images property for all UW jobs
add 6d377a3 Merge pull request #12853: [BEAM-3083] Sets sdk_harness_container_images property for all Dataflow Runner V2 jobs
add 115351e [BEAM-10716] TestPubsub/TestPubsubSignal clean up subscriptions (#12830)
add 9eaef36 Revert "[BEAM-9547] Raise NotImplementedError and WontImplementError throughout DeferredDataframe (#12858)" (#12910)
No new revisions were added by this update.
Summary of changes:
.github/PULL_REQUEST_TEMPLATE.md | 2 +-
.test-infra/jenkins/README.md | 4 +-
.../job_PerformanceTests_InfluxDBIO_IT.groovy | 54 ++
...PostCommit_Python_ValidatesRunner_Flink.groovy} | 10 +-
...b_PreCommit_Python_ValidatesRunner_Flink.groovy | 6 +-
.test-infra/kubernetes/influxdb/influxdb.yml | 76 +++
.test-infra/kubernetes/influxdb/setup.sh | 20 +
.test-infra/kubernetes/influxdb/teardown.sh | 19 +
CHANGES.md | 8 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 6 +-
.../common_transforms/aggregation/lesson-info.yaml | 2 +
.../common_transforms/aggregation/max/cmd/main.go | 43 ++
.../aggregation/max/pkg/task/task.go | 25 +
.../aggregation/{sum => max}/task-info.yaml | 0
.../aggregation/max/task-remote-info.yaml | 2 +
.../go/common_transforms/aggregation/max/task.md | 26 +
.../aggregation/max/test/task_test.go | 44 ++
.../common_transforms/aggregation/min/cmd/main.go | 43 ++
.../aggregation/min/pkg/task/task.go | 25 +
.../aggregation/{sum => min}/task-info.yaml | 0
.../aggregation/min/task-remote-info.yaml | 2 +
.../go/common_transforms/aggregation/min/task.md | 26 +
.../aggregation/min/test/task_test.go | 44 ++
learning/katas/go/course-remote-info.yaml | 2 +-
.../sql/zetasql/ZetaSqlDialectSpecTest.java | 169 ------
.../sdk/extensions/sql/zetasql/ZetaSqlUdfTest.java | 215 +++++++
.../beam/sdk/io/gcp/bigquery/BigQueryServices.java | 4 -
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 3 -
.../gcp/bigquery/BigQueryStorageQuerySource.java | 3 -
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 3 -
.../gcp/bigquery/BigQueryStorageStreamSource.java | 4 -
.../gcp/bigquery/BigQueryStorageTableSource.java | 3 -
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 2 +
.../apache/beam/sdk/io/gcp/pubsub/TestPubsub.java | 6 +
.../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 12 +
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 45 +-
.../beam/sdk/io/gcp/healthcare/FhirIOLROIT.java | 19 +-
sdks/java/io/influxdb/build.gradle | 36 ++
.../beam/sdk/io/influxdb/DBShardInformation.java | 62 ++
.../apache/beam/sdk/io/influxdb/InfluxDbIO.java | 675 +++++++++++++++++++++
.../beam/sdk/io/influxdb/ShardInformation.java | 59 ++
.../io/influxdb/ShardInformationByStartDate.java | 34 +-
.../apache/beam/sdk/io/influxdb/package-info.java | 30 +-
.../apache/beam/sdk/io/influxdb/GenerateData.java | 81 +++
.../sdk/io/influxdb/InfluxDBPipelineOptions.java | 49 ++
.../apache/beam/sdk/io/influxdb/InfluxDbIOIT.java | 376 ++++++++++++
.../beam/sdk/io/influxdb/InfluxDbIOTest.java | 211 +++++++
.../org/apache/beam/sdk/io/influxdb/Model.java | 99 +++
sdks/python/apache_beam/dataframe/frame_base.py | 4 +-
sdks/python/apache_beam/dataframe/frames.py | 9 +-
sdks/python/apache_beam/dataframe/io.py | 167 ++++-
sdks/python/apache_beam/dataframe/io_test.py | 95 +++
.../io/external/snowflake.py} | 24 +-
sdks/python/apache_beam/io/fileio.py | 110 ++--
sdks/python/apache_beam/io/fileio_test.py | 33 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 62 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 51 +-
.../apache_beam/io/gcp/bigquery_io_metadata.py | 16 +-
.../python/apache_beam/options/pipeline_options.py | 43 ++
.../options/pipeline_options_validator_test.py | 19 +
.../runners/dataflow/internal/apiclient.py | 5 +-
.../runners/interactive/background_caching_job.py | 1 -
.../interactive/options/capture_limiters.py | 9 +-
.../interactive/options/capture_limiters_test.py | 16 +-
.../runners/interactive/recording_manager.py | 113 ++--
.../runners/interactive/recording_manager_test.py | 144 ++++-
.../apache_beam/runners/interactive/utils.py | 24 +-
.../python/apache_beam/runners/pipeline_context.py | 43 +-
.../apache_beam/runners/worker/data_plane_test.py | 33 +-
sdks/python/apache_beam/transforms/ptransform.py | 29 +-
sdks/python/apache_beam/transforms/util.py | 2 +-
sdks/python/apache_beam/typehints/decorators.py | 4 +
.../apache_beam/typehints/typed_pipeline_test.py | 58 ++
.../typehints/typed_pipeline_test_py3.py | 32 +
sdks/python/conftest.py | 7 +
sdks/python/mypy.ini | 3 -
sdks/python/scripts/generate_pydoc.sh | 1 +
sdks/python/test-suites/gradle.properties | 6 +-
sdks/python/test-suites/portable/build.gradle | 33 +
settings.gradle | 1 +
.../documentation/io/built-in/google-bigquery.md | 41 +-
.../en/documentation/sdks/python-type-safety.md | 7 +
82 files changed, 3386 insertions(+), 548 deletions(-)
create mode 100644 .test-infra/jenkins/job_PerformanceTests_InfluxDBIO_IT.groovy
rename .test-infra/jenkins/{job_PostCommit_Python35_ValidatesRunner_Flink.groovy => job_PostCommit_Python_ValidatesRunner_Flink.groovy} (77%)
create mode 100644 .test-infra/kubernetes/influxdb/influxdb.yml
create mode 100755 .test-infra/kubernetes/influxdb/setup.sh
create mode 100755 .test-infra/kubernetes/influxdb/teardown.sh
create mode 100644 learning/katas/go/common_transforms/aggregation/max/cmd/main.go
create mode 100644 learning/katas/go/common_transforms/aggregation/max/pkg/task/task.go
copy learning/katas/go/common_transforms/aggregation/{sum => max}/task-info.yaml (100%)
create mode 100644 learning/katas/go/common_transforms/aggregation/max/task-remote-info.yaml
create mode 100644 learning/katas/go/common_transforms/aggregation/max/task.md
create mode 100644 learning/katas/go/common_transforms/aggregation/max/test/task_test.go
create mode 100644 learning/katas/go/common_transforms/aggregation/min/cmd/main.go
create mode 100644 learning/katas/go/common_transforms/aggregation/min/pkg/task/task.go
copy learning/katas/go/common_transforms/aggregation/{sum => min}/task-info.yaml (100%)
create mode 100644 learning/katas/go/common_transforms/aggregation/min/task-remote-info.yaml
create mode 100644 learning/katas/go/common_transforms/aggregation/min/task.md
create mode 100644 learning/katas/go/common_transforms/aggregation/min/test/task_test.go
create mode 100644 sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlUdfTest.java
create mode 100644 sdks/java/io/influxdb/build.gradle
create mode 100644 sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/DBShardInformation.java
create mode 100644 sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java
create mode 100644 sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/ShardInformation.java
copy .test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy => sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/ShardInformationByStartDate.java (50%)
copy .test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy => sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/package-info.java (50%)
create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/GenerateData.java
create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDBPipelineOptions.java
create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOIT.java
create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOTest.java
create mode 100644 sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/Model.java
copy sdks/python/{conftest.py => apache_beam/io/external/snowflake.py} (66%)
create mode 100644 sdks/python/test-suites/portable/build.gradle