You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2018/03/06 19:51:37 UTC
[beam] 01/03: Merge branch 'master' of
https://github.com/apache/beam.git
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit fa0428d1afa71a09582893a31a1ec70cd1f2b515
Merge: 834be30 e3e4e3c
Author: 1028332163 <10...@qq.com>
AuthorDate: Thu Feb 22 21:25:11 2018 +0800
Merge branch 'master' of https://github.com/apache/beam.git
.gitignore | 1 +
.test-infra/jenkins/common_job_properties.groovy | 7 +
.test-infra/jenkins/job_beam_Inventory.groovy | 66 +
.../jenkins/job_beam_PerformanceTests_JDBC.groovy | 76 +-
.../job_beam_ReleaseCandidate_Python.groovy | 51 +
assembly.xml | 1 +
build.gradle | 5 +-
build_rules.gradle | 60 +-
examples/java/pom.xml | 2 +-
.../beam/examples/complete/AutoComplete.java | 3 +
.../beam/examples/cookbook/BigQueryTornadoes.java | 2 +-
.../examples/cookbook/CombinePerKeyExamples.java | 2 +-
.../beam/examples/cookbook/FilterExamples.java | 6 +-
.../beam/examples/cookbook/TriggerExample.java | 7 +-
.../apache/beam/examples/snippets/Snippets.java | 2 +
examples/pom.xml | 2 +-
model/fn-execution/pom.xml | 2 +-
.../fn-execution/src/main/proto/beam_fn_api.proto | 7 +
model/job-management/pom.xml | 2 +-
model/pipeline/pom.xml | 2 +-
model/pom.xml | 2 +-
pom.xml | 13 +-
.../run_release_candidate_python_validation.sh | 292 +
runners/apex/build.gradle | 2 +-
runners/apex/pom.xml | 2 +-
.../operators/ApexProcessFnOperator.java | 2 +-
runners/core-construction-java/build.gradle | 30 +-
runners/core-construction-java/pom.xml | 2 +-
.../runners/core/construction/Environments.java | 22 +
.../core/construction/JavaReadViaImpulse.java | 95 +
.../construction/WindowingStrategyTranslation.java | 1 +
.../core/construction/graph/ExecutableStage.java | 84 +-
.../core/construction/graph/FusedPipeline.java | 38 +
.../graph/GreedilyFusedExecutableStage.java | 233 -
.../graph/GreedyPCollectionFusers.java | 79 +-
.../construction/graph/GreedyPipelineFuser.java | 6 +-
.../core/construction/graph/GreedyStageFuser.java | 178 +
.../graph/ImmutableExecutableStage.java | 52 +
.../core/construction/graph/QueryablePipeline.java | 44 +-
.../InMemoryArtifactStagerService.java | 19 +-
.../core/construction/JavaReadViaImpulseTest.java | 131 +
.../construction/graph/ExecutableStageTest.java | 156 +
.../graph/GreedyPipelineFuserTest.java | 139 +
...bleStageTest.java => GreedyStageFuserTest.java} | 174 +-
.../construction/graph/QueryablePipelineTest.java | 16 +-
runners/core-java/build.gradle | 36 +-
runners/core-java/pom.xml | 7 +-
.../org/apache/beam/runners/core/DoFnRunners.java | 4 +-
.../apache/beam/runners/core/ProcessFnRunner.java | 2 +-
.../runners/core/metrics/MetricsTranslation.java | 138 +
.../core/metrics/MetricsTranslationTest.java | 157 +
runners/direct-java/build.gradle | 90 +-
runners/direct-java/pom.xml | 3 +-
.../beam/runners/direct/DirectRegistrar.java | 1 +
.../runners/direct/ImpulseEvaluatorFactory.java | 104 +
.../beam/runners/direct/RootProviderRegistry.java | 2 +
.../runners/direct/TransformEvaluatorRegistry.java | 2 +
.../direct/ImpulseEvaluatorFactoryTest.java | 120 +
.../direct/UnboundedReadEvaluatorFactoryTest.java | 2 +-
runners/flink/build.gradle | 26 +-
runners/flink/pom.xml | 3 +-
.../beam/runners/flink/FlinkPipelineOptions.java | 12 +
.../flink/FlinkStreamingTransformTranslators.java | 35 +-
.../apache/beam/runners/flink/TestFlinkRunner.java | 1 +
.../wrappers/streaming/DoFnOperator.java | 127 +-
.../wrappers/streaming/SplittableDoFnOperator.java | 16 +
.../wrappers/streaming/WindowDoFnOperator.java | 18 +
.../streaming/io/BoundedSourceWrapper.java | 28 +
.../streaming/io/UnboundedSourceWrapper.java | 56 +-
.../beam/runners/flink/ReadSourceITCase.java | 2 +-
.../runners/flink/ReadSourceStreamingITCase.java | 2 +-
.../flink/streaming/GroupByNullKeyTest.java | 2 +-
.../streaming/TopWikipediaSessionsITCase.java | 2 +-
runners/gcp/gcemd/pom.xml | 2 +-
runners/gcp/gcsproxy/pom.xml | 2 +-
runners/gcp/pom.xml | 2 +-
runners/gearpump/build.gradle | 2 +-
runners/gearpump/pom.xml | 2 +-
runners/google-cloud-dataflow-java/build.gradle | 5 +-
runners/google-cloud-dataflow-java/pom.xml | 4 +-
.../beam/runners/dataflow/DataflowRunner.java | 118 +-
.../beam/runners/dataflow/util/TimeUtil.java | 19 +-
runners/java-fn-execution/pom.xml | 2 +-
.../environment/EnvironmentManager.java | 28 +-
.../fnexecution/environment/RemoteEnvironment.java | 26 +-
.../fnexecution/environment/package-info.java | 19 +-
runners/local-artifact-service-java/pom.xml | 2 +-
runners/local-java/pom.xml | 2 +-
runners/pom.xml | 2 +-
runners/reference/java/pom.xml | 2 +-
runners/reference/job-server/pom.xml | 2 +-
runners/reference/pom.xml | 2 +-
runners/spark/pom.xml | 2 +-
.../beam/runners/spark/io/EmptyCheckpointMark.java | 2 +-
.../spark/translation/MultiDoFnFunction.java | 2 +-
.../spark/translation/SparkContextFactory.java | 2 +-
.../runners/spark/ProvidedSparkContextTest.java | 2 +-
sdks/go/README.md | 98 +
sdks/go/cmd/specialize/main.go | 240 +
sdks/go/cmd/symtab/main.go | 101 +
sdks/go/container/Dockerfile | 30 +
sdks/go/container/boot.go | 127 +
sdks/{python => go}/container/pom.xml | 10 +-
sdks/go/data/haiku/old_pond.txt | 3 +
sdks/go/data/shakespeare/hamlet.txt | 6045 ++++++
sdks/go/data/shakespeare/kinglear-hashtag.txt | 5525 +++++
sdks/go/data/shakespeare/kinglear.txt | 5525 +++++
sdks/go/data/shakespeare/romeoandjuliet.txt | 4766 +++++
.../examples/complete/autocomplete/autocomplete.go | 71 +
sdks/go/examples/contains/contains.go | 99 +
sdks/go/examples/cookbook/combine/combine.go | 106 +
sdks/go/examples/cookbook/filter/filter.go | 106 +
sdks/go/examples/cookbook/join/join.go | 111 +
sdks/go/examples/cookbook/max/max.go | 88 +
sdks/go/examples/cookbook/tornadoes/tornadoes.go | 119 +
.../debugging_wordcount/debugging_wordcount.go | 163 +
sdks/go/examples/forest/forest.go | 81 +
sdks/go/examples/grades/grades.go | 105 +
.../minimal_wordcount/minimal_wordcount.go | 101 +
sdks/go/examples/pingpong/pingpong.go | 131 +
sdks/go/examples/wordcap/wordcap.go | 75 +
sdks/go/examples/wordcount/wordcount.go | 168 +
sdks/go/examples/yatzy/yatzy.go | 139 +
sdks/go/pkg/beam/coder.go | 180 +
.../pkg/beam/{util/grpcx/dial.go => coder_test.go} | 34 +-
sdks/go/pkg/beam/combine.go | 78 +
.../syscall_default.go => core/funcx/doc.go} | 20 +-
sdks/go/pkg/beam/core/funcx/fn.go | 427 +
sdks/go/pkg/beam/core/funcx/fn_test.go | 224 +
sdks/go/pkg/beam/core/funcx/output.go | 76 +
sdks/go/pkg/beam/core/funcx/output_test.go | 48 +
sdks/go/pkg/beam/core/funcx/sideinput.go | 100 +
sdks/go/pkg/beam/core/funcx/sideinput_test.go | 70 +
sdks/go/pkg/beam/core/funcx/signature.go | 212 +
sdks/go/pkg/beam/core/funcx/signature_test.go | 212 +
sdks/go/pkg/beam/core/graph/bind.go | 325 +
sdks/go/pkg/beam/core/graph/bind_test.go | 207 +
sdks/go/pkg/beam/core/graph/coder/coder.go | 298 +
sdks/go/pkg/beam/core/graph/coder/int.go | 71 +
sdks/go/pkg/beam/core/graph/coder/int_test.go | 88 +
.../grpcx/dial.go => core/graph/coder/time.go} | 31 +-
sdks/go/pkg/beam/core/graph/coder/varint.go | 98 +
sdks/go/pkg/beam/core/graph/coder/varint_test.go | 94 +
.../syscall_default.go => core/graph/doc.go} | 20 +-
sdks/go/pkg/beam/core/graph/edge.go | 397 +
sdks/go/pkg/beam/core/graph/fn.go | 304 +
sdks/go/pkg/beam/core/graph/graph.go | 126 +
sdks/go/pkg/beam/core/graph/graph_test.go | 68 +
sdks/go/pkg/beam/core/graph/node.go | 70 +
.../syscall_default.go => core/graph/scope.go} | 27 +-
sdks/go/pkg/beam/core/graph/window/window.go | 71 +
sdks/go/pkg/beam/core/runtime/coderx/int.go | 98 +
sdks/go/pkg/beam/core/runtime/coderx/varint.go | 139 +
.../go/pkg/beam/core/runtime/coderx/varint_test.go | 84 +
sdks/go/pkg/beam/core/runtime/exec/coder.go | 319 +
sdks/go/pkg/beam/core/runtime/exec/cogbk.go | 171 +
sdks/go/pkg/beam/core/runtime/exec/combine.go | 257 +
sdks/go/pkg/beam/core/runtime/exec/combine_test.go | 67 +
sdks/go/pkg/beam/core/runtime/exec/datasink.go | 90 +
sdks/go/pkg/beam/core/runtime/exec/datasource.go | 203 +
sdks/go/pkg/beam/core/runtime/exec/decode.go | 102 +
.../runtime/exec/discard.go} | 41 +-
sdks/go/pkg/beam/core/runtime/exec/emit.go | 126 +
sdks/go/pkg/beam/core/runtime/exec/encode.go | 102 +
sdks/go/pkg/beam/core/runtime/exec/flatten.go | 75 +
sdks/go/pkg/beam/core/runtime/exec/flatten_test.go | 49 +
sdks/go/pkg/beam/core/runtime/exec/fn.go | 201 +
sdks/go/pkg/beam/core/runtime/exec/fn_test.go | 256 +
sdks/go/pkg/beam/core/runtime/exec/fullvalue.go | 145 +
.../pkg/beam/core/runtime/exec/fullvalue_test.go | 72 +
sdks/go/pkg/beam/core/runtime/exec/input.go | 188 +
sdks/go/pkg/beam/core/runtime/exec/multiplex.go | 62 +
.../pkg/beam/core/runtime/exec/multiplex_test.go | 55 +
.../beam/core/runtime/exec/optimized/callers.go | 632 +
.../beam/core/runtime/exec/optimized/callers.tmpl | 69 +
.../beam/core/runtime/exec/optimized/decoders.go | 2407 +++
.../beam/core/runtime/exec/optimized/decoders.tmpl | 146 +
.../beam/core/runtime/exec/optimized/emitters.go | 14215 +++++++++++++
.../beam/core/runtime/exec/optimized/emitters.tmpl | 112 +
.../beam/core/runtime/exec/optimized/encoders.go | 2299 ++
.../beam/core/runtime/exec/optimized/encoders.tmpl | 146 +
.../runtime/exec/optimized/gen.go} | 19 +-
.../pkg/beam/core/runtime/exec/optimized/inputs.go | 21272 +++++++++++++++++++
.../beam/core/runtime/exec/optimized/inputs.tmpl | 181 +
sdks/go/pkg/beam/core/runtime/exec/pardo.go | 193 +
sdks/go/pkg/beam/core/runtime/exec/pardo_test.go | 97 +
sdks/go/pkg/beam/core/runtime/exec/plan.go | 153 +
.../runtime/exec/status.go} | 21 +-
sdks/go/pkg/beam/core/runtime/exec/translate.go | 433 +
sdks/go/pkg/beam/core/runtime/exec/unit.go | 75 +
sdks/go/pkg/beam/core/runtime/exec/unit_test.go | 111 +
sdks/go/pkg/beam/core/runtime/exec/util.go | 72 +
sdks/go/pkg/beam/core/runtime/graphx/coder.go | 415 +
sdks/go/pkg/beam/core/runtime/graphx/coder_test.go | 107 +
sdks/go/pkg/beam/core/runtime/graphx/cogbk.go | 108 +
.../runtime/graphx/doc.go} | 27 +-
sdks/go/pkg/beam/core/runtime/graphx/serialize.go | 962 +
sdks/go/pkg/beam/core/runtime/graphx/translate.go | 402 +
.../pkg/beam/core/runtime/graphx/translate_test.go | 90 +
sdks/go/pkg/beam/core/runtime/graphx/tree.go | 121 +
sdks/go/pkg/beam/core/runtime/graphx/user.go | 110 +
.../runtime/graphx/v1/gen.go} | 14 +-
sdks/go/pkg/beam/core/runtime/graphx/v1/v1.pb.go | 821 +
sdks/go/pkg/beam/core/runtime/graphx/v1/v1.proto | 232 +
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 308 +
.../runtime/harness/gen.go} | 14 +-
sdks/go/pkg/beam/core/runtime/harness/harness.go | 291 +
sdks/go/pkg/beam/core/runtime/harness/init/init.go | 116 +
sdks/go/pkg/beam/core/runtime/harness/logging.go | 151 +
sdks/go/pkg/beam/core/runtime/harness/session.go | 215 +
.../go/pkg/beam/core/runtime/harness/session.proto | 69 +
.../core/runtime/harness/session/session.pb.go | 444 +
sdks/go/pkg/beam/core/runtime/init.go | 44 +
sdks/go/pkg/beam/core/runtime/options.go | 91 +
.../grpcx/dial.go => core/runtime/options_test.go} | 44 +-
sdks/go/pkg/beam/core/runtime/symbols.go | 93 +
sdks/go/pkg/beam/core/runtime/types.go | 62 +
sdks/go/pkg/beam/core/runtime/types_test.go | 74 +
sdks/go/pkg/beam/core/typex/class.go | 174 +
sdks/go/pkg/beam/core/typex/class_test.go | 104 +
sdks/go/pkg/beam/core/typex/fulltype.go | 400 +
sdks/go/pkg/beam/core/typex/fulltype_test.go | 125 +
sdks/go/pkg/beam/core/typex/special.go | 74 +
sdks/go/pkg/beam/core/util/dot/dot.go | 130 +
.../syscall.go => core/util/ioutilx/read.go} | 29 +-
sdks/go/pkg/beam/core/util/protox/any.go | 101 +
sdks/go/pkg/beam/core/util/protox/any_test.go | 83 +
.../grpcx/dial.go => core/util/protox/base64.go} | 37 +-
.../util/protox/protox.go} | 20 +-
.../grpcx/dial.go => core/util/protox/query.go} | 34 +-
sdks/go/pkg/beam/core/util/reflectx/call.go | 120 +
sdks/go/pkg/beam/core/util/reflectx/calls.go | 1341 ++
sdks/go/pkg/beam/core/util/reflectx/calls.tmpl | 72 +
sdks/go/pkg/beam/core/util/reflectx/functions.go | 45 +
.../syscall.go => core/util/reflectx/json.go} | 21 +-
sdks/go/pkg/beam/core/util/reflectx/tags.go | 74 +
sdks/go/pkg/beam/core/util/reflectx/types.go | 111 +
sdks/go/pkg/beam/core/util/symtab/symtab.go | 127 +
sdks/go/pkg/beam/create.go | 108 +
.../beam/{util/grpcx/dial.go => create_test.go} | 38 +-
sdks/go/pkg/beam/doc.go | 67 +
sdks/go/pkg/beam/encoding.go | 148 +
sdks/go/pkg/beam/external.go | 64 +
sdks/go/pkg/beam/flatten.go | 72 +
sdks/go/pkg/beam/forward.go | 94 +
sdks/go/pkg/beam/gbk.go | 104 +
sdks/go/pkg/beam/impulse.go | 46 +
sdks/go/pkg/beam/io/bigqueryio/bigquery.go | 241 +
.../dial.go => io/bigqueryio/bigquery_test.go} | 35 +-
sdks/go/pkg/beam/io/textio/filesystem.go | 49 +
sdks/go/pkg/beam/io/textio/gcs/gcs.go | 130 +
.../grpcx/dial.go => io/textio/local/local.go} | 44 +-
sdks/go/pkg/beam/io/textio/textio.go | 207 +
sdks/go/pkg/beam/log/log.go | 183 +
.../{util/syscallx/syscall.go => log/standard.go} | 26 +-
.../beam/model/fnexecution_v1/beam_fn_api.pb.go | 424 +-
sdks/go/pkg/beam/option.go | 72 +
.../grpcx/dial.go => options/gcpopts/options.go} | 27 +-
sdks/go/pkg/beam/pardo.go | 314 +
sdks/go/pkg/beam/partition.go | 124 +
sdks/go/pkg/beam/partition_test.go | 155 +
sdks/go/pkg/beam/pcollection.go | 90 +
sdks/go/pkg/beam/pipeline.go | 87 +
sdks/go/pkg/beam/runner.go | 51 +
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 367 +
sdks/go/pkg/beam/runners/dataflow/messages.go | 174 +
sdks/go/pkg/beam/runners/dataflow/translate.go | 448 +
sdks/go/pkg/beam/runners/direct/buffer.go | 165 +
sdks/go/pkg/beam/runners/direct/direct.go | 274 +
sdks/go/pkg/beam/runners/direct/gbk.go | 135 +
sdks/go/pkg/beam/runners/direct/impulse.go | 61 +
sdks/go/pkg/beam/runners/dot/dot.go | 55 +
sdks/go/pkg/beam/runners/session/session.go | 337 +
sdks/go/pkg/beam/testing/passert/passert.go | 214 +
sdks/go/pkg/beam/testing/ptest/ptest.go | 59 +
.../dial.go => transforms/filter/distinct.go} | 39 +-
.../go/pkg/beam/transforms/filter/distinct_test.go | 70 +
sdks/go/pkg/beam/transforms/filter/filter.go | 95 +
sdks/go/pkg/beam/transforms/filter/filter_test.go | 90 +
.../syscall.go => transforms/stats/count.go} | 23 +-
sdks/go/pkg/beam/transforms/stats/count_test.go | 69 +
sdks/go/pkg/beam/transforms/stats/max.go | 44 +
sdks/go/pkg/beam/transforms/stats/max_switch.go | 138 +
.../stats/max_switch.tmpl} | 30 +-
sdks/go/pkg/beam/transforms/stats/max_test.go | 134 +
sdks/go/pkg/beam/transforms/stats/mean.go | 91 +
sdks/go/pkg/beam/transforms/stats/mean_test.go | 121 +
sdks/go/pkg/beam/transforms/stats/min.go | 44 +
sdks/go/pkg/beam/transforms/stats/min_switch.go | 138 +
.../stats/min_switch.tmpl} | 30 +-
sdks/go/pkg/beam/transforms/stats/min_test.go | 121 +
sdks/go/pkg/beam/transforms/stats/sum.go | 44 +
sdks/go/pkg/beam/transforms/stats/sum_switch.go | 102 +
.../stats/sum_switch.tmpl} | 27 +-
sdks/go/pkg/beam/transforms/stats/sum_test.go | 117 +
sdks/go/pkg/beam/transforms/stats/util.go | 48 +
sdks/go/pkg/beam/transforms/top/top.go | 180 +
sdks/go/pkg/beam/transforms/top/top_test.go | 126 +
sdks/go/pkg/beam/util.go | 117 +
sdks/go/pkg/beam/util/gcsx/gcs.go | 40 +
sdks/go/pkg/beam/util/grpcx/dial.go | 12 +-
sdks/go/pkg/beam/util/syscallx/syscall.go | 6 +-
sdks/go/pkg/beam/util/syscallx/syscall_default.go | 4 +-
sdks/go/pkg/beam/validate.go | 79 +
sdks/go/pkg/beam/x/beamx/run.go | 41 +
sdks/go/pkg/beam/x/debug/head.go | 68 +
sdks/go/pkg/beam/x/debug/print.go | 97 +
sdks/go/pom.xml | 10 +-
sdks/java/build-tools/pom.xml | 2 +-
.../src/main/resources/beam/checkstyle.xml | 7 +-
sdks/java/container/pom.xml | 2 +-
sdks/java/core/pom.xml | 2 +-
.../java/org/apache/beam/sdk/coders/Coder.java | 2 +-
.../org/apache/beam/sdk/coders/CoderProviders.java | 2 +-
.../org/apache/beam/sdk/coders/IterableCoder.java | 10 +
.../java/org/apache/beam/sdk/coders/RowCoder.java | 4 +-
.../apache/beam/sdk/coders/SerializableCoder.java | 41 +
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 26 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 112 +-
.../apache/beam/sdk/options/PipelineOptions.java | 5 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 345 +-
.../beam/sdk/options/ProxyInvocationHandler.java | 18 +-
.../apache/beam/sdk/testing/CombineFnTester.java | 1 +
.../java/org/apache/beam/sdk/testing/PAssert.java | 19 +
.../apache/beam/sdk/testing/SuccessOrFailure.java | 19 +
.../apache/beam/sdk/testing/WindowFnTestUtils.java | 6 +-
.../beam/sdk/transforms/ApproximateUnique.java | 104 +-
.../org/apache/beam/sdk/transforms/Combine.java | 2 +
.../org/apache/beam/sdk/transforms/CombineFns.java | 18 +
.../java/org/apache/beam/sdk/transforms/DoFn.java | 45 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 2 +-
.../org/apache/beam/sdk/transforms/Impulse.java | 3 +-
.../java/org/apache/beam/sdk/transforms/Wait.java | 2 +-
.../beam/sdk/transforms/reflect/DoFnSignature.java | 4 +
.../transforms/reflect/OnTimerMethodSpecifier.java | 2 +-
.../windowing/MergeOverlappingIntervalWindows.java | 3 +
.../beam/sdk/util/SerializableThrowable.java | 18 +
.../beam/sdk/util/common/ReflectHelpers.java | 48 +
.../java/org/apache/beam/sdk/values/RowType.java | 12 +-
.../apache/beam/sdk/values/TimestampedValue.java | 6 +
.../beam/sdk/coders/SerializableCoderTest.java | 49 +
.../beam/sdk/coders/StructuredCoderTest.java | 3 +-
.../java/org/apache/beam/sdk/io/FileIOTest.java | 75 +
.../org/apache/beam/sdk/io/TextIOReadTest.java | 2 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 74 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 4 +-
.../beam/sdk/testing/CombineFnTesterTest.java | 33 +-
.../sdk/testing/InterceptingUrlClassLoader.java | 13 +-
.../org/apache/beam/sdk/testing/PAssertTest.java | 1 +
.../beam/sdk/transforms/ApproximateUniqueTest.java | 60 +
.../apache/beam/sdk/transforms/CombineFnsTest.java | 16 +
.../apache/beam/sdk/transforms/CombineTest.java | 132 +-
.../apache/beam/sdk/transforms/DistinctTest.java | 61 +
.../apache/beam/sdk/transforms/DoFnTesterTest.java | 1 +
.../org/apache/beam/sdk/transforms/FilterTest.java | 75 +
.../beam/sdk/transforms/FlatMapElementsTest.java | 43 +
.../beam/sdk/transforms/GroupIntoBatchesTest.java | 2 +-
.../beam/sdk/transforms/MapElementsTest.java | 63 +
.../org/apache/beam/sdk/transforms/ParDoTest.java | 4 +-
.../apache/beam/sdk/transforms/PartitionTest.java | 31 +
.../beam/sdk/transforms/SimpleFunctionTest.java | 33 +
.../org/apache/beam/sdk/transforms/WaitTest.java | 69 +-
.../apache/beam/sdk/transforms/WithKeysTest.java | 34 +
.../beam/sdk/transforms/WithTimestampsTest.java | 30 +
.../display/DisplayDataMatchersTest.java | 2 +-
.../sdk/transforms/windowing/WindowingTest.java | 1 +
.../sdk/util/ExposedByteArrayOutputStreamTest.java | 2 +-
.../beam/sdk/values/reflect/RowFactoryTest.java | 6 +-
.../extensions/google-cloud-platform-core/pom.xml | 2 +-
.../sdk/util/RetryHttpRequestInitializerTest.java | 2 +-
sdks/java/extensions/jackson/pom.xml | 2 +-
.../extensions/jackson/JacksonTransformsTest.java | 24 +
sdks/java/extensions/join-library/pom.xml | 2 +-
sdks/java/extensions/pom.xml | 2 +-
sdks/java/extensions/protobuf/pom.xml | 2 +-
sdks/java/extensions/sketching/pom.xml | 2 +-
sdks/java/extensions/sorter/pom.xml | 2 +-
sdks/java/extensions/sql/pom.xml | 2 +-
.../apache/beam/sdk/extensions/sql/BeamSql.java | 1 +
.../sdk/extensions/sql/BeamSqlSeekableTable.java | 10 +
.../apache/beam/sdk/extensions/sql/BeamSqlUdf.java | 2 +-
.../beam/sdk/extensions/sql/QueryTransform.java | 3 +-
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 17 +-
.../sql/impl/interpreter/BeamSqlFnExecutor.java | 4 +
.../operator/BeamSqlDefaultExpression.java} | 28 +-
.../interpreter/operator/BeamSqlPrimitive.java | 2 +-
.../sql/impl/interpreter/operator/UdafImpl.java | 20 +-
.../comparison/BeamSqlIsNotNullExpression.java | 2 +-
.../comparison/BeamSqlIsNullExpression.java | 2 +-
.../date/BeamSqlCurrentDateExpression.java | 1 +
.../date/BeamSqlCurrentTimeExpression.java | 1 +
.../date/BeamSqlCurrentTimestampExpression.java | 1 +
.../operator/date/BeamSqlDateCeilExpression.java | 1 +
.../operator/date/BeamSqlDateFloorExpression.java | 1 +
.../operator/date/BeamSqlExtractExpression.java | 1 +
.../operator/logical/BeamSqlLogicalExpression.java | 1 +
.../sql/impl/planner/BeamQueryPlanner.java | 21 +-
.../extensions/sql/impl/planner/BeamRuleSets.java | 54 +-
.../sql/impl/rel/BeamAggregationRel.java | 19 +-
.../sdk/extensions/sql/impl/rel/BeamFilterRel.java | 34 +-
.../sdk/extensions/sql/impl/rel/BeamIOSinkRel.java | 85 +-
.../extensions/sql/impl/rel/BeamIOSourceRel.java | 45 +-
.../extensions/sql/impl/rel/BeamIntersectRel.java | 33 +-
.../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 328 +-
.../sdk/extensions/sql/impl/rel/BeamMinusRel.java | 16 +-
.../extensions/sql/impl/rel/BeamProjectRel.java | 37 +-
.../sdk/extensions/sql/impl/rel/BeamRelNode.java | 15 +-
.../sql/impl/rel/BeamSetOperatorRelBase.java | 14 +-
.../sdk/extensions/sql/impl/rel/BeamSortRel.java | 71 +-
.../sdk/extensions/sql/impl/rel/BeamUnionRel.java | 15 +-
.../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 40 +-
.../extensions/sql/impl/rule/BeamIOSinkRule.java | 58 +-
.../extensions/sql/impl/rule/BeamIOSourceRule.java | 30 +-
.../sdk/extensions/sql/impl/rule/BeamJoinRule.java | 33 +-
.../extensions/sql/impl/rule/BeamProjectRule.java | 37 +-
.../sql/impl/schema/BeamPCollectionTable.java | 2 +-
.../impl/transform/BeamAggregationTransforms.java | 2 +-
.../sql/impl/transform/BeamJoinTransforms.java | 10 +
.../sql/impl/transform/agg/CovarianceFn.java | 2 +-
.../sql/impl/transform/agg/VarianceFn.java | 2 +-
.../beam/sdk/extensions/sql/BeamSqlDslBase.java | 2 +-
.../sdk/extensions/sql/BeamSqlDslJoinTest.java | 6 +-
.../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 120 +-
.../interpreter/BeamSqlFnExecutorTestBase.java | 42 +-
.../operator/BeamSqlCompareExpressionTest.java | 10 +-
.../operator/BeamSqlInputRefExpressionTest.java | 4 +-
.../interpreter/operator/BeamSqlPrimitiveTest.java | 10 +-
.../sdk/extensions/sql/impl/rel/CheckSize.java | 1 +
.../transform/agg/VarianceAccumulatorTest.java | 6 +-
.../sql/impl/transform/agg/VarianceFnTest.java | 3 +-
.../extensions/sql/mock/MockedBoundedTable.java | 2 +-
.../extensions/sql/mock/MockedUnboundedTable.java | 2 +-
sdks/java/fn-execution/pom.xml | 2 +-
.../org/apache/beam/sdk/fn/stream/DataStreams.java | 3 +-
sdks/java/harness/pom.xml | 2 +-
.../fn/harness/state/StateFetchingIterators.java | 4 +-
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 1 +
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 1 +
sdks/java/io/amazon-web-services/pom.xml | 2 +-
.../apache/beam/sdk/io/aws/s3/S3FileSystem.java | 132 +-
.../beam/sdk/io/aws/s3/S3FileSystemTest.java | 228 +-
sdks/java/io/amqp/pom.xml | 2 +-
sdks/java/io/cassandra/pom.xml | 2 +-
.../beam/sdk/io/cassandra/CassandraIOTest.java | 18 +
sdks/java/io/common/pom.xml | 2 +-
.../org/apache/beam/sdk/io/common/HashingFn.java | 19 +
.../elasticsearch-tests-2/pom.xml | 2 +-
.../elasticsearch-tests-5/pom.xml | 2 +-
.../sdk/io/elasticsearch/ElasticsearchIOTest.java | 8 +-
.../elasticsearch-tests-common/pom.xml | 2 +-
.../io/elasticsearch/ElasticSearchIOTestUtils.java | 4 +-
.../elasticsearch/ElasticsearchIOTestCommon.java | 2 +-
sdks/java/io/elasticsearch-tests/pom.xml | 2 +-
sdks/java/io/elasticsearch/pom.xml | 2 +-
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 13 +-
sdks/java/io/file-based-io-tests/pom.xml | 3 +-
.../java/org/apache/beam/sdk/io/avro/AvroIOIT.java | 4 +-
sdks/java/io/google-cloud-platform/pom.xml | 2 +-
.../beam/sdk/io/gcp/bigquery/BatchLoads.java | 50 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 125 +-
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 26 +-
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 3 +-
.../gcp/bigquery/DynamicDestinationsHelpers.java | 1 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 10 +
.../beam/sdk/io/gcp/bigquery/TableDestination.java | 11 +-
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 27 +-
.../beam/sdk/io/gcp/spanner/BatchSpannerRead.java | 161 +
.../sdk/io/gcp/spanner/CreateTransactionFn.java | 29 +-
.../beam/sdk/io/gcp/spanner/MutationGroup.java | 18 +
.../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 85 -
.../beam/sdk/io/gcp/spanner/OrderedCode.java | 2 +-
.../beam/sdk/io/gcp/spanner/ReadOperation.java | 13 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 9 +-
.../beam/sdk/io/gcp/spanner/SpannerConfig.java | 5 +-
.../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 51 +-
.../beam/sdk/io/gcp/spanner/SpannerSchema.java | 19 +
.../beam/sdk/io/gcp/spanner/Transaction.java | 10 +-
.../cloud/spanner/FakeBatchTransactionId.java | 44 +-
.../google/cloud/spanner/FakePartitionFactory.java | 43 +
.../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 2 +-
.../sdk/io/gcp/bigquery/TableRowJsonCoderTest.java | 1 +
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 2 +-
.../io/gcp/bigtable/BigtableServiceImplTest.java | 2 +-
.../sdk/io/gcp/spanner/FakeServiceFactory.java | 13 +
.../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 329 +-
.../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 42 +
sdks/java/io/hadoop-common/pom.xml | 2 +-
sdks/java/io/hadoop-file-system/pom.xml | 2 +-
sdks/java/io/hadoop-input-format/build.gradle | 7 +
sdks/java/io/hadoop-input-format/pom.xml | 14 +-
.../io/hadoop/inputformat/HadoopInputFormatIO.java | 2 +-
.../inputformat/HadoopInputFormatIOTest.java | 2 +-
sdks/java/io/hbase/pom.xml | 2 +-
sdks/java/io/hcatalog/pom.xml | 2 +-
sdks/java/io/jdbc/pom.xml | 4 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 73 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 18 +-
sdks/java/io/jms/pom.xml | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 2 +-
sdks/java/io/kafka/README.md | 17 +-
sdks/java/io/kafka/pom.xml | 2 +-
.../org/apache/beam/sdk/io/kafka/ConsumerSpEL.java | 20 +-
.../beam/sdk/io/kafka/KafkaCheckpointMark.java | 32 +-
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 134 +-
.../org/apache/beam/sdk/io/kafka/KafkaRecord.java | 14 +-
.../apache/beam/sdk/io/kafka/KafkaRecordCoder.java | 3 +
.../beam/sdk/io/kafka/KafkaTimestampType.java} | 43 +-
.../beam/sdk/io/kafka/KafkaUnboundedReader.java | 200 +-
.../apache/beam/sdk/io/kafka/TimestampPolicy.java | 68 +
.../beam/sdk/io/kafka/TimestampPolicyFactory.java | 206 +
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 151 +-
sdks/java/io/kinesis/pom.xml | 2 +-
.../beam/sdk/io/kinesis/KinesisReaderTest.java | 65 +-
sdks/java/io/mongodb/pom.xml | 2 +-
.../apache/beam/sdk/io/mongodb/MongoDbIOTest.java | 25 +-
sdks/java/io/mqtt/pom.xml | 2 +-
sdks/java/io/pom.xml | 2 +-
sdks/java/io/redis/pom.xml | 2 +-
sdks/java/io/solr/pom.xml | 2 +-
sdks/java/io/tika/pom.xml | 2 +-
sdks/java/io/xml/pom.xml | 2 +-
sdks/java/java8tests/build.gradle | 31 -
sdks/java/java8tests/pom.xml | 89 -
.../options/PipelineOptionsFactoryJava8Test.java | 92 -
.../beam/sdk/transforms/CombineJava8Test.java | 171 -
.../beam/sdk/transforms/DistinctJava8Test.java | 97 -
.../beam/sdk/transforms/FilterJava8Test.java | 114 -
.../sdk/transforms/FlatMapElementsJava8Test.java | 84 -
.../beam/sdk/transforms/MapElementsJava8Test.java | 97 -
.../beam/sdk/transforms/PartitionJava8Test.java | 74 -
.../sdk/transforms/SimpleFunctionJava8Test.java | 68 -
.../beam/sdk/transforms/WithKeysJava8Test.java | 73 -
.../sdk/transforms/WithTimestampsJava8Test.java | 71 -
sdks/java/javadoc/pom.xml | 2 +-
.../maven-archetypes/examples/generate-sources.sh | 26 +-
sdks/java/maven-archetypes/examples/pom.xml | 2 +-
sdks/java/maven-archetypes/pom.xml | 2 +-
sdks/java/maven-archetypes/starter/pom.xml | 2 +-
sdks/java/nexmark/pom.xml | 2 +-
.../org/apache/beam/sdk/nexmark/model/Event.java | 4 +-
sdks/java/pom.xml | 3 +-
sdks/pom.xml | 2 +-
sdks/python/apache_beam/coders/coder_impl.pxd | 2 +
sdks/python/apache_beam/coders/coder_impl.py | 14 +-
sdks/python/apache_beam/coders/coders.py | 9 +-
sdks/python/apache_beam/coders/typecoders.py | 4 +-
.../examples/complete/game/game_stats.py | 4 +-
sdks/python/apache_beam/examples/complete/tfidf.py | 8 +-
.../examples/cookbook/datastore_wordcount.py | 5 +-
.../apache_beam/examples/snippets/snippets.py | 5 +-
.../apache_beam/examples/streaming_wordcount.py | 4 +-
.../apache_beam/examples/windowed_wordcount.py | 4 +-
sdks/python/apache_beam/examples/wordcount.py | 4 +-
.../apache_beam/examples/wordcount_debugging.py | 4 +-
.../python/apache_beam/examples/wordcount_fnapi.py | 4 +-
.../apache_beam/examples/wordcount_minimal.py | 4 +-
sdks/python/apache_beam/io/filebasedsink.py | 4 +-
sdks/python/apache_beam/io/filesystem_test.py | 5 +
sdks/python/apache_beam/io/filesystemio_test.py | 5 +
sdks/python/apache_beam/io/filesystems.py | 18 +-
sdks/python/apache_beam/io/filesystems_test.py | 5 +
.../apache_beam/io/gcp/datastore/v1/helper.py | 5 +-
.../apache_beam/io/gcp/gcsfilesystem_test.py | 5 +
sdks/python/apache_beam/io/gcp/pubsub.py | 180 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 179 +-
.../python/apache_beam/io/hadoopfilesystem_test.py | 5 +
.../io/hdfs_integration_test/Dockerfile | 38 +
.../io/hdfs_integration_test/docker-compose.yml | 60 +
.../hdfs_integration_test/hdfs_integration_test.sh | 40 +
sdks/python/apache_beam/io/localfilesystem_test.py | 5 +
sdks/python/apache_beam/metrics/cells.py | 15 +-
sdks/python/apache_beam/metrics/execution.py | 7 +-
sdks/python/apache_beam/options/value_provider.py | 3 +
.../apache_beam/options/value_provider_test.py | 5 +
sdks/python/apache_beam/pvalue.py | 28 +
sdks/python/apache_beam/runners/common.pxd | 3 +-
sdks/python/apache_beam/runners/common.py | 44 +-
.../runners/dataflow/dataflow_metrics.py | 8 +-
.../runners/dataflow/dataflow_runner.py | 4 +
.../runners/dataflow/internal/apiclient.py | 3 +-
.../runners/dataflow/internal/dependency.py | 4 +-
.../apache_beam/runners/dataflow/internal/names.py | 1 +
.../apache_beam/runners/direct/direct_runner.py | 14 +-
sdks/python/apache_beam/runners/direct/executor.py | 7 +-
.../runners/direct/sdf_direct_runner.py | 17 +-
.../runners/direct/sdf_direct_runner_test.py | 29 +-
.../runners/direct/transform_evaluator.py | 51 +-
.../runners/portability/fn_api_runner.py | 45 +-
.../runners/portability/fn_api_runner_test.py | 19 +-
.../portability/universal_local_runner_test.py | 7 +-
sdks/python/apache_beam/runners/sdf_common.py | 6 +-
.../apache_beam/runners/worker/bundle_processor.py | 46 +-
.../apache_beam/runners/worker/data_plane.py | 4 +-
.../apache_beam/runners/worker/data_plane_test.py | 3 +-
.../apache_beam/runners/worker/sdk_worker.py | 4 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 2 +-
.../runners/worker/sdk_worker_main_test.py | 2 +-
sdks/python/apache_beam/transforms/create_test.py | 5 +
.../python/apache_beam/transforms/cy_combiners.pxd | 10 +
sdks/python/apache_beam/transforms/cy_combiners.py | 97 +
.../apache_beam/transforms/cy_combiners_test.py | 65 +
sdks/python/apache_beam/transforms/display.py | 4 +-
sdks/python/apache_beam/transforms/display_test.py | 6 +-
.../apache_beam/transforms/ptransform_test.py | 3 +-
sdks/python/apache_beam/transforms/trigger_test.py | 3 +-
sdks/python/apache_beam/transforms/util.py | 7 +-
sdks/python/apache_beam/transforms/util_test.py | 5 +
sdks/python/apache_beam/typehints/typecheck.py | 11 +-
.../python/apache_beam/typehints/typehints_test.py | 10 +-
sdks/python/apache_beam/utils/counters.py | 3 +
sdks/python/apache_beam/utils/profiler.py | 9 +-
sdks/python/apache_beam/utils/retry.py | 7 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/container/pom.xml | 2 +-
sdks/python/pom.xml | 2 +-
sdks/python/run_mini_py3lint.sh | 40 +
sdks/python/run_postcommit.sh | 3 +
sdks/python/run_pylint.sh | 17 +
sdks/python/setup.py | 3 +-
sdks/python/tox.ini | 21 +-
settings.gradle | 1 -
621 files changed, 97527 insertions(+), 4176 deletions(-)
--
To stop receiving notification emails like this one, please contact
lcwik@apache.org.