You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by gi...@apache.org on 2020/10/22 02:47:21 UTC
[beam] tag nightly-master updated (6f567d4 -> c185adb)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 6f567d4 (commit)
to c185adb (commit)
from 6f567d4 Correct docstrings in existing version of ContextualTextIO (#13146)
add 6bc49b1 [BEAM-11018] Use metric for Python BigQuery streaming insert API latency logging
add ce5d5a4 move histogram to internal.metrics
add 1666dde add missing __init__.py
add 2da9852 fix error
add b165a8c fix docstring error
add d284779 fix test error
add cd13945 modify the comment
add c56e987 Merge pull request #13017 from ihji/BEAM-11018
add 343fd2a [BEAM-11075] Customization of key size and value size (#13149)
add 339ea77 Change imports of org.codehaus.jackson to com.fasterxml.jackson
add 22902c6 Merge pull request #13158 from sjvanrossum/change-jackson-import
add 99aee6c [BEAM-10787] make external sorter 2048MB memory limit Hadoop only
add 3cfe5a3 Merge pull request #12659: [BEAM-10787] make external sorter 2048MB memory limit Hadoop only
add 252732e Make streaming and "auto" Dataflow ValidatesRunner jobs have different titles
add 44e0c66 Merge pull request #13124: Make streaming and "auto" Dataflow ValidatesRunner jobs have different titles
add bc955de Lengthprefix any input coder for an ProcessBundleDescriptor.
add 5419c3b Merge pull request #13120 from [BEAM-10940] Lengthprefix any input coder for an ProcessBundleDescriptor.
add 9921efc Upgrade checkerframework to 3.7.0
add e61deb7 Upgrade checkerframework gradle plugin to 0.5.11
add 07856e3 Merge pull request #13113: Upgrade checkerframework and gradle plugin
add f87a671 Merge pull request #13153 from [BEAM-11093] Adding BQ snippets for Dynamic Destinations and Time Partitioning
add 415572d [BEAM-11094] Only work with fully expanded PCollections in TransformHierarchy and PTransformReplacements (#13118)
add 6e86c95 Use a newer Python interpreter version as default. (#13140)
add c185adb Delete unneeded PCollections in pipeline_from_stages() (#13100)
No new revisions were added by this update.
Summary of changes:
...PostCommit_Java_ValidatesRunner_Dataflow.groovy | 2 +-
..._Java_ValidatesRunner_Dataflow_streaming.groovy | 2 +-
buildSrc/build.gradle | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 7 +-
.../construction/DeduplicatedFlattenFactory.java | 6 +-
.../construction/EmptyFlattenAsCreateFactory.java | 5 +-
.../beam/runners/core/construction/External.java | 5 +-
.../core/construction/PTransformReplacements.java | 5 +-
.../core/construction/PTransformTranslation.java | 27 +--
.../runners/core/construction/PrimitiveCreate.java | 5 +-
.../core/construction/ReplacementOutputs.java | 47 ++---
.../SingleInputOutputOverrideFactory.java | 5 +-
.../runners/core/construction/SplittableParDo.java | 14 +-
.../construction/SplittableParDoNaiveBounded.java | 5 +-
.../runners/core/construction/TransformInputs.java | 3 +-
.../construction/UnsupportedOverrideFactory.java | 6 +-
.../CreatePCollectionViewTranslationTest.java | 9 +-
.../DeduplicatedFlattenFactoryTest.java | 3 +-
.../EmptyFlattenAsCreateFactoryTest.java | 7 +-
.../core/construction/MorePipelineTest.java | 163 ++++++++++++++++++
.../core/construction/PTransformMatchersTest.java | 7 +-
.../construction/PTransformReplacementsTest.java | 14 +-
.../construction/PTransformTranslationTest.java | 25 ++-
.../core/construction/ParDoTranslationTest.java | 8 +-
.../core/construction/ReplacementOutputsTest.java | 20 ++-
.../core/construction/SdkComponentsTest.java | 44 ++++-
.../SingleInputOutputOverrideFactoryTest.java | 9 +-
.../core/construction/SplittableParDoTest.java | 8 +-
.../construction/TestStreamTranslationTest.java | 8 +-
.../core/construction/TransformInputsTest.java | 10 +-
.../construction/WriteFilesTranslationTest.java | 4 +-
.../core/SplittableParDoViaKeyedWorkItems.java | 5 +-
.../apache/beam/runners/direct/DirectGraph.java | 2 +-
.../runners/direct/KeyedPValueTrackingVisitor.java | 3 +-
.../beam/runners/direct/MultiStepCombine.java | 3 +-
.../beam/runners/direct/ParDoEvaluatorFactory.java | 12 +-
.../runners/direct/ParDoMultiOverrideFactory.java | 4 +-
.../runners/direct/TestStreamEvaluatorFactory.java | 5 +-
.../runners/direct/WriteWithShardingFactory.java | 5 +-
.../beam/runners/direct/CommittedResultTest.java | 5 +-
.../direct/WriteWithShardingFactoryTest.java | 4 +-
.../runners/flink/CreateStreamingFlinkView.java | 5 +-
.../flink/FlinkBatchTransformTranslators.java | 6 +-
.../flink/FlinkBatchTranslationContext.java | 4 +-
.../flink/FlinkStreamingPipelineTranslator.java | 4 +-
.../flink/FlinkStreamingTransformTranslators.java | 8 +-
.../flink/FlinkStreamingTranslationContext.java | 4 +-
.../translation/utils/LookupPipelineVisitor.java | 4 +-
.../FlinkStreamingTransformTranslatorsTest.java | 7 +-
.../runners/flink/FlinkTransformOverridesTest.java | 4 +-
.../dataflow/BatchStatefulParDoOverrides.java | 9 +-
.../dataflow/DataflowPipelineTranslator.java | 19 +--
.../beam/runners/dataflow/DataflowRunner.java | 32 ++--
.../runners/dataflow/GroupIntoBatchesOverride.java | 9 +-
.../dataflow/PrimitiveParDoSingleFactory.java | 10 +-
.../RequiresStableInputParDoOverrides.java | 9 +-
.../runners/dataflow/ReshuffleOverrideFactory.java | 2 +-
.../runners/dataflow/SplittableParDoOverrides.java | 5 +-
.../beam/runners/dataflow/TransformTranslator.java | 6 +-
.../beam/runners/dataflow/DataflowRunnerTest.java | 9 +-
.../dataflow/PrimitiveParDoSingleFactoryTest.java | 13 +-
.../control/ProcessBundleDescriptors.java | 36 ++--
.../control/ProcessBundleDescriptorsTest.java | 101 +++++++++++
.../beam/runners/jet/JetTransformTranslators.java | 23 ++-
.../java/org/apache/beam/runners/jet/Utils.java | 26 +--
.../org/apache/beam/runners/jet/TestJetRunner.java | 3 +-
.../translation/FlattenPCollectionsTranslator.java | 12 +-
.../translation/ParDoBoundMultiTranslator.java | 7 +-
.../translation/SplittableParDoTranslators.java | 5 +-
.../org/apache/beam/runners/spark/SparkRunner.java | 12 +-
.../translation/TranslationContext.java | 4 +-
.../translation/batch/FlattenTranslatorBatch.java | 2 +-
.../translation/batch/ParDoTranslatorBatch.java | 6 +-
.../spark/translation/EvaluationContext.java | 4 +-
.../spark/translation/TransformTranslator.java | 15 +-
.../spark/translation/TranslationUtils.java | 7 +-
.../streaming/StreamingTransformTranslator.java | 6 +-
.../twister2/Twister2TranslationContext.java | 4 +-
.../translators/batch/FlattenTranslatorBatch.java | 3 +-
.../batch/ParDoMultiOutputTranslatorBatch.java | 5 +-
sdks/go/pkg/beam/io/synthetic/source.go | 60 ++++++-
sdks/go/pkg/beam/io/synthetic/source_test.go | 55 ++++++
.../main/java/org/apache/beam/sdk/Pipeline.java | 6 +-
.../apache/beam/sdk/runners/AppliedPTransform.java | 16 +-
.../sdk/runners/PTransformOverrideFactory.java | 5 +-
.../beam/sdk/runners/TransformHierarchy.java | 161 +++++------------
.../java/org/apache/beam/sdk/transforms/View.java | 2 +-
.../apache/beam/sdk/values/PCollectionViews.java | 20 +++
.../org/apache/beam/sdk/values/PValueBase.java | 7 +-
.../java/org/apache/beam/sdk/values/PValues.java | 126 ++++++++++++++
.../org/apache/beam/sdk/values/TaggedPValue.java | 8 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 27 +--
.../beam/sdk/runners/TransformHierarchyTest.java | 15 +-
.../extensions/sorter/BufferedExternalSorter.java | 20 ++-
.../beam/sdk/extensions/sorter/ExternalSorter.java | 15 +-
.../sdk/extensions/sorter/ExternalSorterTest.java | 6 +-
.../sdk/extensions/sql/jdbc/BeamSqlLineIT.java | 4 +-
.../apache_beam/examples/snippets/snippets.py | 29 ++++
.../internal/{gcp => metrics}/__init__.py | 0
sdks/python/apache_beam/internal/metrics/cells.py | 190 +++++++++++++++++++++
.../apache_beam/internal/metrics/cells_test.py | 80 +++++++++
sdks/python/apache_beam/internal/metrics/metric.py | 138 +++++++++++++++
.../apache_beam/internal/metrics/metric_test.py | 52 ++++++
sdks/python/apache_beam/io/gcp/bigquery.py | 49 ++----
sdks/python/apache_beam/io/gcp/bigquery_test.py | 1 -
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 33 ++--
.../apache_beam/io/gcp/bigquery_tools_test.py | 20 ---
sdks/python/apache_beam/metrics/cells.py | 14 +-
sdks/python/apache_beam/metrics/execution.py | 10 +-
sdks/python/apache_beam/metrics/metric.py | 3 +
sdks/python/apache_beam/metrics/metricbase.py | 13 +-
.../python/apache_beam/options/pipeline_options.py | 21 ---
.../portability/fn_api_runner/translations.py | 22 ++-
sdks/python/apache_beam/utils/histogram.py | 71 ++++++--
sdks/python/apache_beam/utils/histogram_test.py | 47 ++++-
.../documentation/io/built-in/google-bigquery.md | 17 +-
116 files changed, 1674 insertions(+), 672 deletions(-)
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/MorePipelineTest.java
create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/values/PValues.java
copy sdks/python/apache_beam/internal/{gcp => metrics}/__init__.py (100%)
create mode 100644 sdks/python/apache_beam/internal/metrics/cells.py
create mode 100644 sdks/python/apache_beam/internal/metrics/cells_test.py
create mode 100644 sdks/python/apache_beam/internal/metrics/metric.py
create mode 100644 sdks/python/apache_beam/internal/metrics/metric_test.py