You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2021/10/05 03:58:31 UTC
[beam] tag nightly-master updated (b9457b7 -> b822366)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from b9457b7 (commit)
to b822366 (commit)
from b9457b7 [BEAM-12513] Schemas and Coders (#15632)
add 8e217ea [BEAM-11516] Upgrade to pylint 2.11.1, fix warnings (#15612)
add 935abae [BEAM-12979, BEAM-11097] Change cache to store ReStreams, clean up to… (#15639)
add 2a3d887 [BEAM-3304, BEAM-12513] Trigger changes and Windowing. (#15644)
add d1f4fe0 Merge pull request #15618 from [BEAM-12968] [Playground] Create README file
add 684af19 Merge pull request #15626 from [BEAM-12963] [Playground] Create base Beam Playground page structure
add 6103589 Merge pull request #15566 from [BEAM-12925] Correct behavior retrieving NULL values from JDBC.
add 686efb3 [BEAM-12911] Update schema translation (Java, Python) to log more helpful errors when there are issues with individual fields (#15538)
add 9d1acdc Update CHANGES.md for JdbcIO breaking change (#15651)
add ce605d1 [BEAM-12513] Add Go SDK metrics content to BPG. (#15650)
add e891d24 [BEAM-12996] Improve Error Logging in ConfigBuilder (#15646)
add b822366 [BEAM-13000] Disable Reshuffle Translation in Samza Portable Mode (#15655)
No new revisions were added by this update.
Summary of changes:
.pre-commit-config.yaml | 2 +-
CHANGES.md | 1 +
learning/katas/python/log_elements.py | 4 +-
playground/backend/README.md | 101 ++++++++++++++++
.../lib/components/logo/logo_component.dart | 33 +++---
.../toggle_theme_button/toggle_theme_button.dart} | 54 ++++-----
playground/frontend/lib/config/theme.dart | 86 ++++++++++++++
.../frontend/lib/constants/colors.dart | 24 ++--
.../frontend/lib/constants/sizes.dart | 19 +--
playground/frontend/lib/main.dart | 34 +-----
.../modules/editor/components/editor_textarea.dart | 13 +--
.../lib/modules/output/components/output_area.dart | 13 ++-
.../lib/modules/sdk/components/sdk_selector.dart | 67 +++++++++++
.../frontend/lib/modules/sdk/models/sdk.dart | 29 +++--
.../lib/pages/playground/playground_page.dart | 64 ++++++++++
.../lib/pages/playground/playground_state.dart | 14 +--
.../lib/{main.dart => playground_app.dart} | 44 +++----
playground/frontend/pubspec.lock | 22 ++--
playground/frontend/pubspec.yaml | 6 +-
.../playground/playground_state_test.dart} | 19 +--
playground/frontend/test/widget_test.dart | 3 +-
.../core/construction/SchemaTranslationTest.java | 31 +++++
.../runners/samza/translation/ConfigBuilder.java | 11 +-
.../samza/translation/ReshuffleTranslator.java | 7 +-
.../SamzaPortablePipelineTranslator.java | 3 +-
sdks/go/examples/snippets/08windowing.go | 94 +++++++++++++++
sdks/go/examples/snippets/09triggers.go | 81 ++++++++++---
sdks/go/examples/snippets/10metrics.go | 84 ++++++++++++++
.../snippets/10metrics_test.go} | 27 ++---
sdks/go/pkg/beam/core/graph/window/trigger.go | 129 +++++++++++++++++----
sdks/go/pkg/beam/core/runtime/exec/data.go | 14 ++-
sdks/go/pkg/beam/core/runtime/exec/pardo.go | 1 +
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 34 ++++--
sdks/go/pkg/beam/core/runtime/harness/harness.go | 1 +
.../core/runtime/harness/statecache/statecache.go | 27 ++---
.../runtime/harness/statecache/statecache_test.go | 75 ++++++------
sdks/go/pkg/beam/core/runtime/harness/statemgr.go | 2 +-
sdks/go/pkg/beam/metrics.go | 12 +-
sdks/go/pkg/beam/option.go | 6 +-
sdks/go/pkg/beam/windowing.go | 59 +++++++---
sdks/go/test/integration/primitives/windowinto.go | 46 ++++++--
.../apache/beam/sdk/schemas/SchemaTranslation.java | 8 +-
.../org/apache/beam/sdk/io/jdbc/SchemaUtil.java | 7 +-
.../apache/beam/sdk/io/jdbc/SchemaUtilTest.java | 27 ++++-
sdks/python/.pylintrc | 6 +
sdks/python/apache_beam/coders/coder_impl.py | 5 +-
sdks/python/apache_beam/coders/coders.py | 12 +-
sdks/python/apache_beam/coders/coders_test.py | 2 +-
.../apache_beam/coders/coders_test_common.py | 13 +--
sdks/python/apache_beam/coders/row_coder.py | 6 +-
sdks/python/apache_beam/coders/row_coder_test.py | 12 ++
sdks/python/apache_beam/coders/slow_stream.py | 2 +-
.../apache_beam/coders/standard_coders_test.py | 8 +-
sdks/python/apache_beam/dataframe/doctests.py | 17 ++-
sdks/python/apache_beam/dataframe/doctests_test.py | 1 +
sdks/python/apache_beam/dataframe/expressions.py | 10 +-
sdks/python/apache_beam/dataframe/frame_base.py | 4 +-
sdks/python/apache_beam/dataframe/frames.py | 2 +-
sdks/python/apache_beam/dataframe/io.py | 2 +-
sdks/python/apache_beam/dataframe/io_test.py | 2 +-
.../dataframe/pandas_top_level_functions.py | 2 +-
sdks/python/apache_beam/dataframe/partitionings.py | 1 -
.../apache_beam/dataframe/partitionings_test.py | 2 +-
sdks/python/apache_beam/dataframe/transforms.py | 2 +-
sdks/python/apache_beam/examples/avro_bitcoin.py | 2 +-
.../apache_beam/examples/complete/autocomplete.py | 2 +-
.../examples/complete/game/game_stats.py | 6 +-
.../examples/complete/game/hourly_team_score.py | 8 +-
.../examples/complete/game/leader_board.py | 10 +-
.../examples/complete/game/user_score.py | 4 +-
.../examples/complete/top_wikipedia_sessions.py | 2 +-
.../examples/cookbook/bigtableio_it_test.py | 2 +-
.../apache_beam/examples/fastavro_it_test.py | 4 +-
.../examples/flink/flink_streaming_impulse.py | 2 +-
.../apache_beam/examples/snippets/snippets.py | 4 +-
.../apache_beam/examples/snippets/snippets_test.py | 2 +-
.../apache_beam/examples/streaming_wordcount.py | 2 +-
.../examples/streaming_wordcount_debugging.py | 2 +-
.../apache_beam/examples/windowed_wordcount.py | 2 +-
.../apache_beam/examples/wordcount_debugging.py | 2 +-
.../apache_beam/examples/wordcount_it_test.py | 3 +-
.../apache_beam/examples/wordcount_with_metrics.py | 2 +-
sdks/python/apache_beam/internal/gcp/auth.py | 3 +-
sdks/python/apache_beam/internal/metrics/metric.py | 6 +-
sdks/python/apache_beam/io/avroio.py | 10 +-
sdks/python/apache_beam/io/avroio_test.py | 6 +-
sdks/python/apache_beam/io/aws/s3filesystem.py | 2 +-
sdks/python/apache_beam/io/concat_source.py | 4 +-
.../apache_beam/io/external/generate_sequence.py | 2 +-
.../io/external/xlang_kafkaio_it_test.py | 2 +-
.../io/external/xlang_kinesisio_it_test.py | 2 +-
.../io/external/xlang_parquetio_test.py | 2 +-
.../io/external/xlang_snowflakeio_it_test.py | 2 +-
sdks/python/apache_beam/io/filebasedsink_test.py | 4 +-
sdks/python/apache_beam/io/fileio_test.py | 4 +-
sdks/python/apache_beam/io/filesystem.py | 2 +-
sdks/python/apache_beam/io/filesystem_test.py | 4 +-
sdks/python/apache_beam/io/filesystemio.py | 2 +-
sdks/python/apache_beam/io/gcp/__init__.py | 2 +-
.../apache_beam/io/gcp/bigquery_read_perf_test.py | 2 +-
.../apache_beam/io/gcp/bigquery_write_perf_test.py | 2 +-
sdks/python/apache_beam/io/gcp/bigtableio.py | 6 +-
.../io/gcp/datastore/v1new/datastoreio.py | 10 +-
.../io/gcp/datastore/v1new/query_splitter.py | 4 +-
.../io/gcp/datastore/v1new/rampup_throttling_fn.py | 2 +-
.../apache_beam/io/gcp/experimental/spannerio.py | 2 +-
.../gcp/experimental/spannerio_read_perf_test.py | 2 +-
.../gcp/experimental/spannerio_write_perf_test.py | 2 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 1 +
sdks/python/apache_beam/io/gcp/gcsio.py | 2 +-
.../apache_beam/io/gcp/gcsio_integration_test.py | 4 +-
.../clients/bigquery/bigquery_v2_client.py | 18 +--
.../internal/clients/storage/storage_v1_client.py | 20 ++--
sdks/python/apache_beam/io/gcp/pubsub.py | 8 +-
.../apache_beam/io/gcp/pubsub_io_perf_test.py | 8 +-
sdks/python/apache_beam/io/gcp/spanner.py | 2 +-
.../apache_beam/io/gcp/tests/bigquery_matcher.py | 6 +-
.../io/gcp/tests/xlang_spannerio_it_test.py | 2 +-
sdks/python/apache_beam/io/hadoopfilesystem.py | 2 +-
sdks/python/apache_beam/io/iobase.py | 10 +-
sdks/python/apache_beam/io/iobase_test.py | 10 +-
sdks/python/apache_beam/io/jdbc.py | 4 +-
sdks/python/apache_beam/io/kafka.py | 4 +-
sdks/python/apache_beam/io/kinesis.py | 4 +-
sdks/python/apache_beam/io/mongodbio.py | 5 +-
sdks/python/apache_beam/io/parquetio.py | 16 +--
sdks/python/apache_beam/io/parquetio_it_test.py | 2 +-
sdks/python/apache_beam/io/range_trackers.py | 2 +-
sdks/python/apache_beam/io/restriction_trackers.py | 2 +-
.../apache_beam/io/source_test_utils_test.py | 2 +-
sdks/python/apache_beam/io/textio.py | 19 ++-
sdks/python/apache_beam/io/textio_test.py | 4 +-
sdks/python/apache_beam/io/tfrecordio.py | 10 +-
sdks/python/apache_beam/metrics/cells.py | 6 +-
sdks/python/apache_beam/metrics/execution.py | 4 +-
sdks/python/apache_beam/metrics/metric.py | 6 +-
.../python/apache_beam/metrics/monitoring_infos.py | 4 +-
.../apache_beam/ml/gcp/videointelligenceml.py | 8 +-
sdks/python/apache_beam/ml/gcp/visionml.py | 6 +-
.../python/apache_beam/options/pipeline_options.py | 4 +-
sdks/python/apache_beam/pipeline.py | 12 +-
sdks/python/apache_beam/pipeline_test.py | 4 +-
sdks/python/apache_beam/pvalue.py | 4 +-
sdks/python/apache_beam/runners/common.py | 6 +-
.../runners/dataflow/dataflow_metrics.py | 4 +-
.../runners/dataflow/dataflow_runner.py | 4 +-
.../runners/dataflow/dataflow_runner_test.py | 2 +-
.../runners/dataflow/internal/apiclient.py | 7 +-
.../runners/dataflow/internal/apiclient_test.py | 6 +-
.../clients/dataflow/dataflow_v1b3_client.py | 44 +++----
.../runners/dataflow/native_io/iobase.py | 2 +-
.../runners/dataflow/ptransform_overrides.py | 4 +-
.../runners/dataflow/test_dataflow_runner.py | 3 +-
.../consumer_tracking_pipeline_visitor_test.py | 10 +-
.../apache_beam/runners/direct/direct_runner.py | 8 +-
.../apache_beam/runners/direct/direct_userstate.py | 12 +-
.../runners/direct/evaluation_context.py | 2 +-
sdks/python/apache_beam/runners/direct/executor.py | 8 +-
.../runners/direct/sdf_direct_runner_test.py | 2 +-
.../runners/direct/test_direct_runner.py | 2 +-
.../runners/direct/transform_evaluator.py | 32 ++---
.../runners/interactive/augmented_pipeline.py | 4 +-
.../runners/interactive/background_caching_job.py | 2 +-
.../runners/interactive/cache_manager.py | 2 +-
.../display/interactive_pipeline_graph.py | 2 +-
.../display/pcoll_visualization_test.py | 2 +-
.../interactive/display/pipeline_graph_test.py | 2 +-
.../runners/interactive/interactive_beam_test.py | 10 +-
.../interactive/interactive_environment_test.py | 4 +-
.../runners/interactive/interactive_runner.py | 4 +-
.../runners/interactive/interactive_runner_test.py | 2 +-
.../interactive_environment_inspector_test.py | 10 +-
.../runners/interactive/pipeline_fragment.py | 2 +-
.../runners/interactive/pipeline_fragment_test.py | 8 +-
.../interactive/pipeline_instrument_test.py | 16 +--
.../testing/integration/notebook_executor.py | 2 +-
.../interactive/testing/integration/screen_diff.py | 4 +-
.../testing/integration/tests/screen_diff_test.py | 5 +-
.../interactive/testing/pipeline_assertion.py | 5 +-
.../apache_beam/runners/interactive/utils.py | 4 +-
.../runners/portability/abstract_job_service.py | 2 +-
.../runners/portability/artifact_service.py | 1 +
.../runners/portability/flink_runner.py | 4 +-
.../runners/portability/flink_runner_test.py | 21 ++--
.../portability/flink_uber_jar_job_server.py | 6 +-
.../runners/portability/fn_api_runner/fn_runner.py | 10 +-
.../portability/fn_api_runner/fn_runner_test.py | 9 +-
.../portability/fn_api_runner/trigger_manager.py | 2 +-
.../portability/fn_api_runner/worker_handlers.py | 25 ++--
.../apache_beam/runners/portability/job_server.py | 2 +-
.../runners/portability/local_job_service.py | 9 +-
.../runners/portability/portable_runner.py | 2 +-
.../runners/portability/portable_runner_test.py | 12 +-
.../runners/portability/samza_runner_test.py | 5 +-
.../runners/portability/spark_runner.py | 4 +-
.../runners/portability/spark_runner_test.py | 5 +-
.../portability/spark_uber_jar_job_server.py | 6 +-
.../portability/spark_uber_jar_job_server_test.py | 1 +
sdks/python/apache_beam/runners/runner.py | 2 +-
.../apache_beam/runners/worker/bundle_processor.py | 15 ++-
.../apache_beam/runners/worker/data_plane.py | 13 +--
.../apache_beam/runners/worker/log_handler.py | 4 +-
sdks/python/apache_beam/runners/worker/logger.py | 4 +-
.../apache_beam/runners/worker/opcounters.py | 4 +-
.../apache_beam/runners/worker/operations.py | 1 +
.../apache_beam/runners/worker/sideinputs_test.py | 2 +-
.../apache_beam/runners/worker/statesampler.py | 14 +--
.../benchmarks/nexmark/queries/winning_bids.py | 2 +-
.../testing/load_tests/co_group_by_key_test.py | 2 +-
.../apache_beam/testing/load_tests/combine_test.py | 2 +-
.../testing/load_tests/group_by_key_test.py | 2 +-
.../testing/load_tests/load_test_metrics_utils.py | 10 +-
.../testing/load_tests/microbenchmarks_test.py | 2 +-
.../apache_beam/testing/load_tests/pardo_test.py | 2 +-
.../testing/load_tests/sideinput_test.py | 2 +-
.../apache_beam/testing/metric_result_matchers.py | 4 +-
.../testing/metric_result_matchers_test.py | 2 +-
sdks/python/apache_beam/testing/test_pipeline.py | 4 +-
sdks/python/apache_beam/testing/test_stream.py | 2 +-
sdks/python/apache_beam/testing/test_utils.py | 6 +-
sdks/python/apache_beam/testing/util.py | 6 +-
.../tools/fn_api_runner_microbenchmark.py | 2 +-
.../apache_beam/tools/teststream_microbenchmark.py | 2 +-
.../transforms/combinefn_lifecycle_pipeline.py | 2 +-
sdks/python/apache_beam/transforms/combiners.py | 19 ++-
sdks/python/apache_beam/transforms/core.py | 20 ++--
.../apache_beam/transforms/deduplicate_test.py | 2 +-
sdks/python/apache_beam/transforms/external.py | 6 +-
.../python/apache_beam/transforms/external_test.py | 4 +-
.../apache_beam/transforms/periodicsequence.py | 4 +-
sdks/python/apache_beam/transforms/ptransform.py | 25 ++--
.../apache_beam/transforms/ptransform_test.py | 4 +-
sdks/python/apache_beam/transforms/sql.py | 2 +-
sdks/python/apache_beam/transforms/stats_test.py | 2 +-
sdks/python/apache_beam/transforms/trigger.py | 6 +-
sdks/python/apache_beam/transforms/userstate.py | 2 +-
sdks/python/apache_beam/transforms/util.py | 9 +-
sdks/python/apache_beam/transforms/window.py | 2 +-
sdks/python/apache_beam/transforms/window_test.py | 2 +-
sdks/python/apache_beam/typehints/schemas.py | 21 +++-
sdks/python/apache_beam/typehints/schemas_test.py | 13 +++
.../apache_beam/typehints/trivial_inference.py | 2 +-
sdks/python/apache_beam/typehints/typecheck.py | 6 +-
sdks/python/apache_beam/typehints/typehints.py | 10 +-
.../python/apache_beam/typehints/typehints_test.py | 2 +-
sdks/python/apache_beam/utils/counters.py | 6 +-
sdks/python/apache_beam/utils/histogram.py | 2 +-
sdks/python/apache_beam/utils/profiler.py | 2 +-
sdks/python/apache_beam/utils/shared.py | 4 +-
sdks/python/apache_beam/utils/subprocess_server.py | 6 +-
.../apache_beam/utils/thread_pool_executor.py | 2 +-
sdks/python/gen_protos.py | 3 +-
sdks/python/setup.py | 2 +-
sdks/python/tox.ini | 4 +-
.../en/documentation/patterns/cross-language.md | 2 +-
.../content/en/documentation/programming-guide.md | 125 ++++++++++++++++++--
256 files changed, 1801 insertions(+), 933 deletions(-)
create mode 100644 playground/backend/README.md
copy learning/katas/java/IO/Built-in IOs/Built-in IOs/src/org/apache/beam/learning/katas/io/builtinios/Task.java => playground/frontend/lib/components/logo/logo_component.dart (63%)
copy playground/frontend/lib/{main.dart => components/toggle_theme_button/toggle_theme_button.dart} (51%)
create mode 100644 playground/frontend/lib/config/theme.dart
copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfo.java => playground/frontend/lib/constants/colors.dart (58%)
copy learning/katas/java/IO/Built-in IOs/Built-in IOs/test/org/apache/beam/learning/katas/io/builtinios/TaskTest.java => playground/frontend/lib/constants/sizes.dart (74%)
copy sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/data/text/SnowflakeChar.java => playground/frontend/lib/modules/editor/components/editor_textarea.dart (76%)
copy sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/data/datetime/SnowflakeDateTime.java => playground/frontend/lib/modules/output/components/output_area.dart (76%)
create mode 100644 playground/frontend/lib/modules/sdk/components/sdk_selector.dart
copy sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/ProjectSupport.java => playground/frontend/lib/modules/sdk/models/sdk.dart (72%)
create mode 100644 playground/frontend/lib/pages/playground/playground_page.dart
copy sdks/java/io/snowflake/src/main/java/org/apache/beam/sdk/io/snowflake/data/text/SnowflakeChar.java => playground/frontend/lib/pages/playground/playground_state.dart (78%)
copy playground/frontend/lib/{main.dart => playground_app.dart} (62%)
copy playground/frontend/test/{widget_test.dart => pages/playground/playground_state_test.dart} (64%)
create mode 100644 sdks/go/examples/snippets/08windowing.go
create mode 100644 sdks/go/examples/snippets/10metrics.go
copy sdks/go/{pkg/beam/io/textio/sdf_test.go => examples/snippets/10metrics_test.go} (62%)