You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jo...@apache.org on 2023/06/16 16:50:34 UTC
[beam] branch master updated (7ecd8e51444 -> 218e4b3b2dd)
This is an automated email from the ASF dual-hosted git repository.
johncasey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 7ecd8e51444 Adding error tag and metrics in SpannerWriteSchemaTransformProvider (#27021)
add 218e4b3b2dd .test-infra Pipeline reads from Dataflow API and writes to BigQuery (#27008)
No new revisions were added by this update.
Summary of changes:
.gitignore | 3 +
.test-infra/pipelines/README.md | 34 ++
.test-infra/pipelines/build.gradle | 85 +++++
.../infrastructure/01.setup/.terraform.lock.hcl | 40 ++
.../pipelines/infrastructure/01.setup/README.md | 83 +++++
.../01.setup/apache-beam-testing.tfbackend | 19 +
.../01.setup/apache-beam-testing.tfvars | 19 +
.../infrastructure/01.setup/artifactregistry.tf | 30 ++
.../infrastructure/01.setup/common.tfvars | 21 ++
.../pipelines/infrastructure/01.setup/iam.tf | 37 ++
.../infrastructure/01.setup}/provider.tf | 0
.../pipelines/infrastructure/01.setup/services.tf | 35 ++
.../pipelines/infrastructure/01.setup/state.tf | 26 ++
.../pipelines/infrastructure/01.setup/storage.tf | 43 +++
.../pipelines/infrastructure/01.setup/variables.tf | 37 ++
.../infrastructure/02.network/.terraform.lock.hcl | 22 ++
.../pipelines/infrastructure/02.network/README.md | 81 ++++
.../02.network/apache-beam-testing.tfbackend | 19 +
.../02.network/apache-beam-testing.tfvars | 19 +
.../infrastructure/02.network/common.tfvars | 22 ++
.../pipelines/infrastructure/02.network/data.tf | 22 ++
.../pipelines/infrastructure/02.network/network.tf | 46 +++
.../infrastructure/02.network}/provider.tf | 0
.../pipelines/infrastructure/02.network/state.tf | 26 ++
.../infrastructure/02.network/variables.tf | 42 +++
.../pipelines/infrastructure/03.io/README.md | 24 ++
.../03.io/dataflow-to-bigquery/.terraform.lock.hcl | 21 ++
.../03.io/dataflow-to-bigquery/README.md | 86 +++++
.../apache-beam-testing.tfbackend | 19 +
.../apache-beam-testing.tfvars | 19 +
.../03.io/dataflow-to-bigquery/bigquery.tf | 30 ++
.../03.io/dataflow-to-bigquery/common.tfvars | 22 ++
.../03.io/dataflow-to-bigquery/data.tf | 27 ++
.../03.io/dataflow-to-bigquery/iam.tf | 41 +++
.../03.io/dataflow-to-bigquery/provider.tf | 21 ++
.../03.io/dataflow-to-bigquery/pubsub.tf | 39 ++
.../03.io/dataflow-to-bigquery/state.tf | 26 ++
.../03.io/dataflow-to-bigquery/variables.tf | 42 +++
.../03.io/dataflow-to-bigquery/workflow.tf | 67 ++++
.../04.template/dataflow-to-bigquery/README.md | 66 ++++
.../apache-beam-testing.tfvars | 20 +
.../04.template/dataflow-to-bigquery/common.tfvars | 24 ++
.../04.template/dataflow-to-bigquery/data.tf | 53 +++
.../dataflow-to-bigquery/dataflow-template.json | 20 +
.../04.template/dataflow-to-bigquery/output.tf | 22 ++
.../04.template/dataflow-to-bigquery/provider.tf | 21 ++
.../04.template/dataflow-to-bigquery/template.tf | 58 +++
.../04.template/dataflow-to-bigquery/variables.tf | 57 +++
.test-infra/pipelines/infrastructure/README.md | 65 ++++
.../pipelines/ReadDataflowApiWriteBigQuery.java | 362 ++++++++++++++++++
.../pipelines/bigquery/BigQueryWriteOptions.java | 35 ++
.../pipelines/bigquery/BigQueryWrites.java | 166 +++++++++
.../bigquery/DatasetReferenceOptionValue.java | 58 +++
.../testinfra/pipelines/bigquery/package-info.java | 20 +
.../pipelines/conversions/ConversionError.java | 73 ++++
.../pipelines/conversions/EventarcConversions.java | 116 ++++++
.../testinfra/pipelines/conversions/JobsToRow.java | 89 +++++
.../pipelines/conversions/RowConversionResult.java | 81 ++++
.../conversions/WithAppendedDetailsToRow.java | 265 +++++++++++++
.../pipelines/conversions/package-info.java | 20 +
.../pipelines/dataflow/DataflowClientFactory.java | 102 +++++
.../DataflowClientFactoryConfiguration.java | 54 +++
.../dataflow/DataflowFilterEventarcJobs.java | 110 ++++++
.../dataflow/DataflowGetJobExecutionDetails.java | 160 ++++++++
.../pipelines/dataflow/DataflowGetJobMetrics.java | 139 +++++++
.../pipelines/dataflow/DataflowGetJobs.java | 121 ++++++
.../dataflow/DataflowGetStageExecutionDetails.java | 166 +++++++++
.../pipelines/dataflow/DataflowJobsOptions.java | 36 ++
.../pipelines/dataflow/DataflowReadResult.java | 90 +++++
.../pipelines/dataflow/DataflowRequestError.java | 95 +++++
.../pipelines/dataflow/DataflowRequests.java | 52 +++
.../dataflow/JobMetricsWithAppendedDetails.java | 84 +++++
.../dataflow/StageSummaryWithAppendedDetails.java | 84 +++++
.../testinfra/pipelines/dataflow/Throttle.java | 74 ++++
.../dataflow/WorkerDetailsWithAppendedDetails.java | 84 +++++
.../testinfra/pipelines/dataflow/package-info.java | 20 +
.../beam/testinfra/pipelines/package-info.java | 19 +
.../pipelines/pubsub/PubsubReadOptions.java | 35 ++
.../pubsub/SubscriptionPathOptionValue.java | 45 +++
.../testinfra/pipelines/pubsub/package-info.java | 20 +
.../schemas/DependencyDrivenDescriptorQueue.java | 110 ++++++
.../schemas/DescriptorSchemaRegistry.java | 153 ++++++++
.../schemas/GeneratedMessageV3RowBuilder.java | 280 ++++++++++++++
.../testinfra/pipelines/schemas/package-info.java | 20 +
.../conversions/EventarcConversionsTest.java | 271 ++++++++++++++
.../JobMetricsWithAppendedDetailsTest.java | 143 +++++++
.../StageSummaryWithAppendedDetailsTest.java | 146 ++++++++
.../conversions/WithAppendedDetailsToRowTest.java | 153 ++++++++
.../WorkerDetailsWithAppendedDetailsTest.java | 153 ++++++++
.../dataflow/DataflowFilterEventarcJobsTest.java | 145 ++++++++
.../AbstractGeneratedMessageV3RowBuilderTest.java | 409 +++++++++++++++++++++
.../DependencyDrivenDescriptorQueueTest.java | 56 +++
.../schemas/DescriptorSchemaRegistryTest.java | 79 ++++
.../schemas/EnvironmentRowBuilderTest.java | 140 +++++++
.../schemas/ExecutionStageSummaryTest.java | 84 +++++
.../schemas/TransformSummaryRowBuilderTest.java | 83 +++++
.../has_extra_data_payload_foo_property.json | 1 +
.../job_state_canceled_streaming.json | 1 +
.../job_state_canceling_streaming.json | 1 +
.../eventarc_data/job_state_done_batch.json | 1 +
.../eventarc_data/job_state_pending_batch.json | 1 +
.../eventarc_data/job_state_pending_streaming.json | 1 +
.../eventarc_data/job_state_queued_streaming.json | 1 +
.../eventarc_data/job_state_running_batch.json | 1 +
.../eventarc_data/job_state_running_streaming.json | 1 +
.../src/main/resources/beam/suppressions.xml | 1 +
settings.gradle.kts | 2 +
107 files changed, 7013 insertions(+)
create mode 100644 .test-infra/pipelines/README.md
create mode 100644 .test-infra/pipelines/build.gradle
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/.terraform.lock.hcl
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/README.md
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/apache-beam-testing.tfbackend
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/apache-beam-testing.tfvars
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/artifactregistry.tf
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/common.tfvars
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/iam.tf
copy .test-infra/{terraform/google-cloud-platform/google-kubernetes-engine/modules/01-setup => pipelines/infrastructure/01.setup}/provider.tf (100%)
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/services.tf
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/state.tf
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/storage.tf
create mode 100644 .test-infra/pipelines/infrastructure/01.setup/variables.tf
create mode 100644 .test-infra/pipelines/infrastructure/02.network/.terraform.lock.hcl
create mode 100644 .test-infra/pipelines/infrastructure/02.network/README.md
create mode 100644 .test-infra/pipelines/infrastructure/02.network/apache-beam-testing.tfbackend
create mode 100644 .test-infra/pipelines/infrastructure/02.network/apache-beam-testing.tfvars
create mode 100644 .test-infra/pipelines/infrastructure/02.network/common.tfvars
create mode 100644 .test-infra/pipelines/infrastructure/02.network/data.tf
create mode 100644 .test-infra/pipelines/infrastructure/02.network/network.tf
copy .test-infra/{terraform/google-cloud-platform/google-kubernetes-engine/modules/02-network => pipelines/infrastructure/02.network}/provider.tf (100%)
create mode 100644 .test-infra/pipelines/infrastructure/02.network/state.tf
create mode 100644 .test-infra/pipelines/infrastructure/02.network/variables.tf
create mode 100644 .test-infra/pipelines/infrastructure/03.io/README.md
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/.terraform.lock.hcl
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/README.md
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/apache-beam-testing.tfbackend
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/apache-beam-testing.tfvars
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/bigquery.tf
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/common.tfvars
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/data.tf
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/iam.tf
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/provider.tf
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/pubsub.tf
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/state.tf
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/variables.tf
create mode 100644 .test-infra/pipelines/infrastructure/03.io/dataflow-to-bigquery/workflow.tf
create mode 100644 .test-infra/pipelines/infrastructure/04.template/dataflow-to-bigquery/README.md
create mode 100644 .test-infra/pipelines/infrastructure/04.template/dataflow-to-bigquery/apache-beam-testing.tfvars
create mode 100644 .test-infra/pipelines/infrastructure/04.template/dataflow-to-bigquery/common.tfvars
create mode 100644 .test-infra/pipelines/infrastructure/04.template/dataflow-to-bigquery/data.tf
create mode 100644 .test-infra/pipelines/infrastructure/04.template/dataflow-to-bigquery/dataflow-template.json
create mode 100644 .test-infra/pipelines/infrastructure/04.template/dataflow-to-bigquery/output.tf
create mode 100644 .test-infra/pipelines/infrastructure/04.template/dataflow-to-bigquery/provider.tf
create mode 100644 .test-infra/pipelines/infrastructure/04.template/dataflow-to-bigquery/template.tf
create mode 100644 .test-infra/pipelines/infrastructure/04.template/dataflow-to-bigquery/variables.tf
create mode 100644 .test-infra/pipelines/infrastructure/README.md
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/ReadDataflowApiWriteBigQuery.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/BigQueryWriteOptions.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/BigQueryWrites.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/DatasetReferenceOptionValue.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/bigquery/package-info.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/ConversionError.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/EventarcConversions.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/JobsToRow.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/RowConversionResult.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/WithAppendedDetailsToRow.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/package-info.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowClientFactory.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowClientFactoryConfiguration.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowFilterEventarcJobs.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobExecutionDetails.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobMetrics.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetJobs.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowGetStageExecutionDetails.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowJobsOptions.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowReadResult.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowRequestError.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowRequests.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/JobMetricsWithAppendedDetails.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/StageSummaryWithAppendedDetails.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/Throttle.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/WorkerDetailsWithAppendedDetails.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/dataflow/package-info.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/package-info.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/pubsub/PubsubReadOptions.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/pubsub/SubscriptionPathOptionValue.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/pubsub/package-info.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DependencyDrivenDescriptorQueue.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/DescriptorSchemaRegistry.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/GeneratedMessageV3RowBuilder.java
create mode 100644 .test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/schemas/package-info.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/EventarcConversionsTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/JobMetricsWithAppendedDetailsTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/StageSummaryWithAppendedDetailsTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/WithAppendedDetailsToRowTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/conversions/WorkerDetailsWithAppendedDetailsTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/dataflow/DataflowFilterEventarcJobsTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/AbstractGeneratedMessageV3RowBuilderTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/DependencyDrivenDescriptorQueueTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/DescriptorSchemaRegistryTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/EnvironmentRowBuilderTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/ExecutionStageSummaryTest.java
create mode 100644 .test-infra/pipelines/src/test/java/org/apache/beam/testinfra/pipelines/schemas/TransformSummaryRowBuilderTest.java
create mode 100644 .test-infra/pipelines/src/test/resources/eventarc_data/has_extra_data_payload_foo_property.json
create mode 100644 .test-infra/pipelines/src/test/resources/eventarc_data/job_state_canceled_streaming.json
create mode 100644 .test-infra/pipelines/src/test/resources/eventarc_data/job_state_canceling_streaming.json
create mode 100644 .test-infra/pipelines/src/test/resources/eventarc_data/job_state_done_batch.json
create mode 100644 .test-infra/pipelines/src/test/resources/eventarc_data/job_state_pending_batch.json
create mode 100644 .test-infra/pipelines/src/test/resources/eventarc_data/job_state_pending_streaming.json
create mode 100644 .test-infra/pipelines/src/test/resources/eventarc_data/job_state_queued_streaming.json
create mode 100644 .test-infra/pipelines/src/test/resources/eventarc_data/job_state_running_batch.json
create mode 100644 .test-infra/pipelines/src/test/resources/eventarc_data/job_state_running_streaming.json