You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2018/01/08 17:25:45 UTC
[beam] branch go-sdk updated (e6f3af2 -> f68f05e)
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a change to branch go-sdk
in repository https://gitbox.apache.org/repos/asf/beam.git.
from e6f3af2 Add builtin varint coder
add 510415e Add grpcio upper bound
add 762efc4 Merge pull request #4273 from aaltay/aaltay-patch-1
add 049070b [BEAM-3189] Sdk worker multithreading (#4134)
add 3dfff97 [BEAM-2101] Allow anonymous access of GCS blobs. (#4254)
add c50d3e9 reshuffle for AvroIOIT and TFRecordIOIT
add 0e79826 Merge pull request #4260 from DariuszAniszewski/reshuffle-for-other-filebased-ioit
add 95d92d4 [BEAM-1920] Upgrade to Spark runner to Spark 2.2.1
add 3d99389 Merge pull request #4208 from jbonofre/BEAM-1920-SPARK2
add 36495b8 [BEAM-3340] Update Flink Runner to Flink 1.4.0
add a424c64 [BEAM-3143] Type Inference Python 3 Compatibility (#4183)
add 9d9ffa5 [BEAM-2929] Remove Dataflow expansions for PCollectionView that have been migrated into the Dataflow service for the portability framework.
add 26560d0 Guard against closing data channel twice (#4283)
add 8103010 Fix getitem and list comprehension type inference.
add 1acd1ae Merge pull request #4282 from robertwb/type-inference
add 8cf222d [BEAM-3183] Add runner.run(transform) to Python SDK.
add 1ff54bb Merge pull request #4247 from robertwb/runner-run
add cd186a5 Fix global match test in localfilesystem
add f7af9ae [BEAM-2865] Introduces FileIO.write() and adds support for it in AvroIO and TextIO
add 173208c Addresses comments
add 6709419 Merge pull request #3817 from jkff/fileio-write
add 76d32e2 [BEAM-3369] Fix typehint errors introduced in PR #4282.
add 0aa943d Merge pull request #4292 from robertwb/BEAM-3369
add 550d6a5 [BEAM-3369] Allow any coder to specify its deterministic variant.
add 157787e Merge pull request #4293 from robertwb/BEAM-3369
add 5443406 Adds support for processing Splittable DoFns using DirectRunner.
add 3c81b41 Merge pull request #4064 from chamikaramj/sdf_direct_runner_3
add 26c5acb [BEAM-3174] Don't parse the release version just directly use the get_distribution version result
add 12db37b [BEAM-3357] Remove upper bound for protobuf (#4274)
add 78ac4da job for performance tests of file-based IOs
add 64b2ef8 make it possible to trigger this job from PR
add ac4e3aa Merge pull request #4267 from DariuszAniszewski/jenkins_job_for_filebased_ioit
add 542c46c Moves out common configuration parts from BigtableIO.Read and BigtableIO.Write to a BigtableConfig class.
add 64c90ac Merge pull request #4277 from dmytroivanov4206/master
add c4945d0 [BEAM-3041] Support various pip based dependency options for python sdk (#4286)
add d0e911f FIX: remove overriding Google project in file-based IOs performance tests
add b4f2ff2 Merge pull request #4296 from DariuszAniszewski/hotfix-for-filebased-io-performance-tests
add a91bd96 Makes gradle findbugs plugin produce an html report rather than xml
add 2063781 Merge pull request #4295 from jkff/findbugs-html
add e92f718 [BEAM-2937] Basic PGBK combiner lifting. (#4290)
add d468d4a Fix for side inputs following flatten/gbk. (#4281)
add 4721fd7 Set min version of protobof so that the range is compatible with grpc dependencies.
add 68d5926 Merge pull request #4298 from chamikaramj/update_min_protobuf_version
add 4570fac [BEAM-3357] Add generated pb2 files to mvn clean. (#4287)
add f2af50a Update wordcount fnapi example to use now supported save_main_session flag
add 38a61c1 Merge pull request #4299 from aaltay/cont
add c396d69 Initial implementation of HadoopFileSsytem.
add 4188278 Merge pull request #4233 from udim/hdfs_py
add 853f0dc Update BigTable client version
add 02e4fb1 Merge pull request #4303 from chamikaramj/update_bigtable_version
add 56f3d0b Remove semicolon
add fa7a36b [BEAM-3060] explicitly use Apache's Google project for file-based performance tests
add d184a33 Merge pull request #4304 from DariuszAniszewski/patch-2
add 6126169 Increment the Dataflow container version
add dfd16f2 [BEAM-981] Add parameter allowing adding jars to spark context
add 57310a6 Post code review amendments and refactor
add cd7ef91 Remove @JsonIgnore to allow setting filesToStage from command line
add 6bf325e Post code review amendments
add 9ad40dc [BEAM-981] Add parameter allowing adding jars to spark context
add 93dee08 Revert "Update BigTable client version"
add 875f3e1 Translate Flink batch nodes by URN
add 8b750e1 Splits BigQueryIOTest into a read and a write test
add b65890e Some cleanups in write test
add 112b725 Some cleanups in read test too
add 117b37a Merge pull request #4309 from jkff/bqio-simplifications
add 151ae67 Made replaceTransforms protected so that DataflowRunner subclass can properly setup overrides.
add f3c498b Enable multiprocess docs generation
add 28c92e4 Merge pull request #4302 from udim/docs
add a4e9446 Update dependency.py
add 37c0ffc Merge pull request #4316 from tvalentyn/patch-3
add 3d1b83c Upgrade oauth2cleitn and google-apitools dependencies
add 255eb9b Merge pull request #4320 from aaltay/apit
add 40bf559 Make release versions of Python sdks use containers tagged with the version of the SDK.
add 95e1751 Merge pull request #4325 from tvalentyn/all_released_sdks_use_containers_tagged_with_beam_version
add cc24c71 Remove extraneous logging.
add aaa0ae6 Merge pull request #4313 from robertwb/robertwb-patch-1
add 383e848 Use pip download instead of the deprecated pip install --download
add 741febd Merge pull request #4321 from aaltay/down
add 549549e Add unit tests to check default values of worker harness container image.
add 41aff80 Merge pull request #4327 from tvalentyn/tests
add 18c4162 Address review comment
add 312c4ab Merge pull request #4300 from mariapython/direct-timers-test
add 660465f Allow Python postcommits to run on beam{1,2,3}.
add d264cf9 Fixes a typo in type signature of TextIO.TypedWrite.to(DynamicDestinations) that made it unusable
add 45a45be Disable incremental compilation for SQL module
add 0be081e Remove repeated slf4j-api dependency from java-fn-execution module
add a0e6025 Merge pull request #4330 from iemejia/fix-sql-compilation-issue
add 792cd17 Set utf-16 as SQL DSL default charset
add 845e22f Merge pull request #4258 from Alienero/master
add 6a0b787 enable large-scale test for FileBasedIOIT
add 24c69aa Merge pull request #4318 from DariuszAniszewski/large-scale-test-for-filebasedioit
add 12d9448 use dedicated BigQuery table for performance tests of FileBasedIOIT
add 70044ae Merge pull request #4317 from DariuszAniszewski/proper_bq_table_for_filebasedioit
add 6450a9c [SQL] Fix shading configuration
add be3c4a6 Fix typechecking for FluentIterable use.
add db4e80c Added ProxyInfoFromEnvironmentVar() & GetNewHttp() methods to get proxy environment settings.
add 2ca4bac Added parsing HTTP_PROXY variable in uppercase as well.
add 0b5b691 Some nit-picks around lower cases.
add 97719ce Adding example to show http_proxy format and accept only lower case environment variable names.
add c4dd7d5 Added warnings about malformed proxy URL Removed initialising ProxyInfo variable with Nones
add 71ef05d Reformatted function names from camel case to underscores. https://www.python.org/dev/peps/pep-0008/#function-names
add 8079800 Cleaned function get_new_http, removing unnecessary arguments, kwargs and http intermediate variable.
add ceddb98 Changed environment variable format checks using regexp (re.match)
add 6684754 Initialised proxy_info = None
add b2101f3 Added extra newline before Returns section
add 92e3e37 Wrap to 80 chars for lint
add b5ecd50 Lint - Added two blank lines
add c48f0de Merged changes from gcsio
add 9d6beab Shortening condition.
add 1b50f8b Reverting splitting comments longer than 80 characters to keep them in the same line.
add 3820f45 Returning None and removing unnecessary if condition.
add a341542 Correct os.environ.get call Fix Docstring conventions as per PEP 257
add d3ae98e Fixing trailing whitespace
add 69758b0 Merge pull request #4136 from dsdinter/python-proxy
add 180e19b Avoid importing testing hierarchy for Javadoc comments.
add 694c050 Clean data receive queue
add 5a72437 Merge pull request #4315 from angoenka/clean_receive_queue
add 9c34d59 Update Website Precommits to new RVM instructions
add 604bf8b Merge pull request #4068: Update Website Precommits to new RVM instructions
add b3fd585 Build unshaded JARs.
add 7d9bf07 Correctly handle pass-through "composites" in the Fn Api Runner. (#4326)
add db3d01f [BEAM-3228] Add fix to prevent crashing forked VM during running Kinesis tests
add 5b6fe77 Merge pull request #4337 from aromanenko-dev/BEAM-3228-KinesisMockReadTest
add 21cd637 [Nexmark][SQL] Implement sql query 0
add 0892e23 Merge pull request #4128: [BEAM-3181][Nexmark][SQL] Implement query0
add 88357a2 Supports a sink for XmlIO
add 4248226 Adds TFRecordIO.sink()
add 4fef942 Merge pull request #4294 from jkff/other-sinks
add 8188db4 Use beam-master-$DATE pattern for default Dataflow containers in unreleased BEAM SDK.
add fe46663 Update dependency.py
add eaf4918 fixup: lint
add d28cc55 Merge pull request #4322 from tvalentyn/patch-4
add 8445048 Updates common.py due to a Cython bug.
add 63e2696 Merge pull request #4342 from chamikaramj/beam_sdf_cython_fix
add a59b767 Plumb user metrics through Fn API.
add d2690fa Merge pull request #4344 from robertwb/user-metrics
add d7aaf9b Remove Queue based 60 seconds timeout for GCS io.
add 4f94713 Merge pull request #4324 from aaltay/gcsi
add eb892e9 Clean mock import
add e215b38 Merge pull request #4354 from mariapython/import_fix
add f842c16 [BEAM-3187] Ensure that teardown is called in case of Exception on start/finish
add c537375 [BEAM-3187] Enable PardoLifecycleTest for the Spark runner
add 3bca776 Merge pull request #4130 from iemejia/BEAM-3187-dofn-spark
add ba0cfac [BEAM-3404] Update KinesisIO to use AWS SDK 1.11.255 and KCL 1.8.8
add 2403fc5 [BEAM-3404] Fix and enable KinesisReaderIT
add 5a61174 Merge pull request #4338 from iemejia/BEAM-3404-update-kinesisio
add 944091e [BEAM-3411] Fixing KeyError when on progress update request
add d2ec896 Merge pull request #4355 from angoenka/progress_report_fix_3411
add 35846e3 [BEAM-3373] Add serviceEndpoint parameter to KinesisIO
add 192b4c7 Merge pull request #4358 from iemejia/BEAM-3373-add-serviceEndpoint
new f68f05e Merge pull request #4350: Integrate master into go-sdk
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 1 +
...job_beam_PerformanceTests_FileBasedIO_IT.groovy | 77 +
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 2 +-
.../job_beam_PostCommit_Python_Verify.groovy | 2 +-
.../job_beam_PreCommit_Website_Merge.groovy | 7 +-
.../job_beam_PreCommit_Website_Stage.groovy | 4 +-
.../jenkins/job_beam_PreCommit_Website_Test.groovy | 4 +-
build.gradle | 14 +-
build_rules.gradle | 6 +
examples/java/build.gradle | 2 +-
examples/java/pom.xml | 6 +-
examples/java8/pom.xml | 6 +-
.../fn-execution/src/main/proto/beam_fn_api.proto | 37 +-
pom.xml | 16 +-
.../core/construction/PipelineResources.java | 57 +
.../core/construction/PipelineResourcesTest.java | 76 +
.../core/metrics/MetricsContainerStepMap.java | 19 +
runners/flink/build.gradle | 14 +-
runners/flink/pom.xml | 16 +-
.../flink/FlinkBatchTransformTranslators.java | 27 +-
.../beam/runners/flink/FlinkPipelineOptions.java | 1 -
.../org/apache/beam/runners/flink/FlinkRunner.java | 36 +-
runners/google-cloud-dataflow-java/pom.xml | 19 +-
.../dataflow/DataflowPipelineTranslator.java | 37 +-
.../beam/runners/dataflow/DataflowRunner.java | 62 +-
.../options/DataflowPipelineWorkerPoolOptions.java | 1 -
.../beam/runners/dataflow/DataflowRunnerTest.java | 36 -
runners/java-fn-execution/pom.xml | 5 -
runners/spark/build.gradle | 5 +-
runners/spark/pom.xml | 96 +-
.../beam/runners/spark/SparkContextOptions.java | 2 +-
.../beam/runners/spark/SparkPipelineOptions.java | 14 +-
.../org/apache/beam/runners/spark/SparkRunner.java | 13 +
.../runners/spark/io/SparkUnboundedSource.java | 6 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 5 +-
.../spark/stateful/SparkTimerInternals.java | 11 +-
.../spark/translation/MultiDoFnFunction.java | 5 +-
.../spark/translation/SparkContextFactory.java | 5 +
.../SparkGroupAlsoByWindowViaOutputBufferFn.java | 5 +-
.../spark/translation/SparkProcessContext.java | 68 +-
.../spark/translation/TranslationUtils.java | 39 +-
.../runners/spark/util/GlobalWatermarkHolder.java | 24 +-
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 137 +-
.../apache/beam/sdk/io/DefaultFilenamePolicy.java | 2 +-
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 52 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 997 +++++++-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 36 +
.../main/java/org/apache/beam/sdk/io/TextIO.java | 140 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 11 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 6 +-
.../org/apache/beam/sdk/options/ValueProvider.java | 3 +-
.../apache/beam/sdk/options/ValueProviders.java | 4 +-
.../apache/beam/sdk/transforms/Requirements.java | 10 +
.../beam/sdk/transforms/SerializableFunctions.java | 20 +-
.../java/org/apache/beam/sdk/util/ShardedFile.java | 3 +-
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 275 ++-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 137 +-
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 87 +-
.../extensions/google-cloud-platform-core/pom.xml | 18 +
sdks/java/extensions/sql/build.gradle | 11 +-
sdks/java/extensions/sql/pom.xml | 1 +
.../sql/impl/planner/BeamQueryPlanner.java | 10 +
.../beam/sdk/extensions/sql/BeamSqlDslBase.java | 2 +-
.../sdk/extensions/sql/BeamSqlNonAsciiTest.java | 57 +
.../data/BeamFnDataBufferingOutboundObserver.java | 9 +
.../BeamFnDataBufferingOutboundObserverTest.java | 19 +
sdks/java/harness/pom.xml | 17 +
.../fn/harness/control/ProcessBundleHandler.java | 1 -
.../java/org/apache/beam/sdk/io/avro/AvroIOIT.java | 4 +-
.../apache/beam/sdk/io/tfrecord/TFRecordIOIT.java | 4 +-
sdks/java/io/google-cloud-platform/pom.xml | 17 +
.../beam/sdk/io/gcp/bigtable/BigtableConfig.java | 261 ++
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 478 ++--
.../sdk/io/gcp/bigquery/BigQueryHelpersTest.java | 139 ++
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 787 ++++++
.../beam/sdk/io/gcp/bigquery/BigQueryIOTest.java | 2546 +-------------------
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 1388 +++++++++++
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 51 +-
.../beam/sdk/io/gcp/bigquery/FakeJobService.java | 20 +-
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 27 +-
sdks/java/io/hadoop/jdk1.8-tests/build.gradle | 1 +
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 24 +-
.../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 +-
sdks/java/io/jdbc/pom.xml | 4 +-
.../io/kafka/serialization/InstantSerializer.java | 2 +-
sdks/java/io/kinesis/build.gradle | 6 +-
sdks/java/io/kinesis/pom.xml | 27 +-
.../org/apache/beam/sdk/io/kinesis/KinesisIO.java | 67 +-
.../apache/beam/sdk/io/kinesis/KinesisReader.java | 2 +-
.../beam/sdk/io/kinesis/AmazonKinesisMock.java | 46 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 86 +-
.../beam/sdk/io/kinesis/KinesisUploader.java | 28 +-
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 265 +-
.../java/org/apache/beam/sdk/io/xml/XmlSink.java | 156 --
.../java/org/apache/beam/sdk/io/xml/XmlIOTest.java | 270 +++
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 280 ---
.../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 87 +-
sdks/java/javadoc/pom.xml | 6 +-
.../src/main/resources/archetype-resources/pom.xml | 6 +-
.../src/main/resources/archetype-resources/pom.xml | 6 +-
sdks/java/nexmark/build.gradle | 6 +
sdks/java/nexmark/pom.xml | 29 +-
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 121 +-
.../apache/beam/sdk/nexmark/NexmarkOptions.java | 7 +
.../org/apache/beam/sdk/nexmark/model/Auction.java | 8 +-
.../org/apache/beam/sdk/nexmark/model/Person.java | 6 +-
.../beam/sdk/nexmark/model/sql/BeamRecordSize.java | 134 ++
.../beam/sdk/nexmark/model/sql/ToBeamRecord.java | 82 +
.../model/sql/adapter/ModelAdaptersMapping.java | 124 +
.../model/sql/adapter/ModelFieldsAdapter.java} | 26 +-
.../sql/adapter}/package-info.java | 6 +-
.../generator => model/sql}/package-info.java | 6 +-
.../beam/sdk/nexmark/queries/NexmarkQuery.java | 5 +-
.../{Query9.java => sql/NexmarkSqlQuery.java} | 33 +-
.../beam/sdk/nexmark/queries/sql/SqlQuery0.java | 97 +
.../sdk/nexmark/queries/sql}/package-info.java | 8 +-
.../sdk/nexmark/model/sql/BeamRecordSizeTest.java | 112 +
.../sdk/nexmark/model/sql/ToBeamRecordTest.java | 109 +
.../sql/adapter/ModelAdaptersMappingTest.java | 159 ++
.../sdk/nexmark/queries/sql/SqlQuery0Test.java | 70 +
sdks/python/apache_beam/coders/coders.py | 40 +
sdks/python/apache_beam/coders/typecoders.py | 11 +-
.../apache_beam/examples/snippets/snippets_test.py | 4 +-
.../python/apache_beam/examples/wordcount_fnapi.py | 11 +-
sdks/python/apache_beam/internal/gcp/auth.py | 8 +-
sdks/python/apache_beam/io/filebasedsource.py | 2 +-
sdks/python/apache_beam/io/filesystem.py | 4 +-
sdks/python/apache_beam/io/filesystems.py | 1 +
sdks/python/apache_beam/io/gcp/gcsio.py | 99 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 38 -
sdks/python/apache_beam/io/hadoopfilesystem.py | 285 +++
.../python/apache_beam/io/hadoopfilesystem_test.py | 472 ++++
sdks/python/apache_beam/io/iobase.py | 15 +-
sdks/python/apache_beam/io/localfilesystem_test.py | 2 +-
sdks/python/apache_beam/io/range_trackers.py | 44 +-
sdks/python/apache_beam/io/range_trackers_test.py | 37 -
sdks/python/apache_beam/io/restriction_trackers.py | 131 +
.../apache_beam/io/restriction_trackers_test.py | 159 ++
sdks/python/apache_beam/metrics/cells.py | 13 +-
sdks/python/apache_beam/metrics/execution.py | 24 +
sdks/python/apache_beam/pipeline.py | 4 +-
sdks/python/apache_beam/pipeline_test.py | 14 +-
sdks/python/apache_beam/runners/common.pxd | 34 +-
sdks/python/apache_beam/runners/common.py | 160 +-
.../runners/dataflow/dataflow_runner.py | 4 +-
.../runners/dataflow/internal/apiclient_test.py | 131 +-
.../runners/dataflow/internal/dependency.py | 60 +-
.../apache_beam/runners/direct/direct_runner.py | 25 +-
sdks/python/apache_beam/runners/direct/executor.py | 3 +
.../runners/direct/sdf_direct_runner.py | 354 +++
.../runners/direct/sdf_direct_runner_test.py | 235 ++
.../runners/direct/transform_evaluator.py | 102 +-
.../runners/portability/fn_api_runner.py | 547 +++--
.../runners/portability/fn_api_runner_test.py | 73 +-
.../runners/portability/maptask_executor_runner.py | 4 +-
.../runners/portability/universal_local_runner.py | 2 +-
sdks/python/apache_beam/runners/runner.py | 17 +-
sdks/python/apache_beam/runners/runner_test.py | 12 +-
sdks/python/apache_beam/runners/sdf_common.py | 168 ++
.../apache_beam/runners/worker/bundle_processor.py | 65 +-
.../apache_beam/runners/worker/data_plane.py | 62 +-
.../apache_beam/runners/worker/operations.py | 9 +-
.../apache_beam/runners/worker/sdk_worker.py | 175 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 38 +-
.../runners/worker/sdk_worker_main_test.py | 33 +-
.../apache_beam/runners/worker/sdk_worker_test.py | 68 +-
sdks/python/apache_beam/transforms/core.py | 24 +-
sdks/python/apache_beam/transforms/trigger.py | 3 +
sdks/python/apache_beam/typehints/opcodes.py | 51 +-
.../apache_beam/typehints/trivial_inference.py | 53 +-
.../typehints/trivial_inference_test.py | 38 +-
sdks/python/apache_beam/utils/urns.py | 3 +
sdks/python/container/boot.go | 57 +-
sdks/python/container/piputil.go | 131 +
sdks/python/gen_protos.py | 3 +-
sdks/python/generate_pydoc.sh | 7 +-
sdks/python/setup.py | 15 +-
sdks/python/tox.ini | 2 +-
settings.gradle | 4 +-
179 files changed, 10480 insertions(+), 4904 deletions(-)
create mode 100644 .test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy
create mode 100644 runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineResources.java
create mode 100644 runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/PipelineResourcesTest.java
create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/BeamSqlNonAsciiTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableConfig.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpersTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadTest.java
create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java
delete mode 100644 sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSink.java
create mode 100644 sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlIOTest.java
delete mode 100644 sdks/java/io/xml/src/test/java/org/apache/beam/sdk/io/xml/XmlSinkTest.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/sql/BeamRecordSize.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/sql/ToBeamRecord.java
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/sql/adapter/ModelAdaptersMapping.java
copy sdks/java/{extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/schema/BaseBeamTable.java => nexmark/src/main/java/org/apache/beam/sdk/nexmark/model/sql/adapter/ModelFieldsAdapter.java} (62%)
copy sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/{sources/generator => model/sql/adapter}/package-info.java (81%)
copy sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/{sources/generator => model/sql}/package-info.java (86%)
copy sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/{Query9.java => sql/NexmarkSqlQuery.java} (52%)
create mode 100644 sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery0.java
copy {runners/apex/src/main/java/org/apache/beam/runners/apex/translation/operators => sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/sql}/package-info.java (86%)
create mode 100644 sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/model/sql/BeamRecordSizeTest.java
create mode 100644 sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/model/sql/ToBeamRecordTest.java
create mode 100644 sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/model/sql/adapter/ModelAdaptersMappingTest.java
create mode 100644 sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/queries/sql/SqlQuery0Test.java
create mode 100644 sdks/python/apache_beam/io/hadoopfilesystem.py
create mode 100644 sdks/python/apache_beam/io/hadoopfilesystem_test.py
create mode 100644 sdks/python/apache_beam/io/restriction_trackers.py
create mode 100644 sdks/python/apache_beam/io/restriction_trackers_test.py
create mode 100644 sdks/python/apache_beam/runners/direct/sdf_direct_runner.py
create mode 100644 sdks/python/apache_beam/runners/direct/sdf_direct_runner_test.py
create mode 100644 sdks/python/apache_beam/runners/sdf_common.py
create mode 100644 sdks/python/container/piputil.go
--
To stop receiving notification emails like this one, please contact
['"commits@beam.apache.org" <co...@beam.apache.org>'].
[beam] 01/01: Merge pull request #4350: Integrate master into go-sdk
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a commit to branch go-sdk
in repository https://gitbox.apache.org/repos/asf/beam.git
commit f68f05ed3487f676d243879f1d7720d91040053d
Merge: e6f3af2 192b4c7
Author: Kenn Knowles <ke...@kennknowles.com>
AuthorDate: Mon Jan 8 09:25:41 2018 -0800
Merge pull request #4350: Integrate master into go-sdk
.gitignore | 1 +
...job_beam_PerformanceTests_FileBasedIO_IT.groovy | 77 +
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 2 +-
.../job_beam_PostCommit_Python_Verify.groovy | 2 +-
.../job_beam_PreCommit_Website_Merge.groovy | 7 +-
.../job_beam_PreCommit_Website_Stage.groovy | 4 +-
.../jenkins/job_beam_PreCommit_Website_Test.groovy | 4 +-
build.gradle | 14 +-
build_rules.gradle | 6 +
examples/java/build.gradle | 2 +-
examples/java/pom.xml | 6 +-
examples/java8/pom.xml | 6 +-
.../fn-execution/src/main/proto/beam_fn_api.proto | 37 +-
pom.xml | 16 +-
.../core/construction/PipelineResources.java | 57 +
.../core/construction/PipelineResourcesTest.java | 76 +
.../core/metrics/MetricsContainerStepMap.java | 19 +
runners/flink/build.gradle | 14 +-
runners/flink/pom.xml | 16 +-
.../flink/FlinkBatchTransformTranslators.java | 27 +-
.../beam/runners/flink/FlinkPipelineOptions.java | 1 -
.../org/apache/beam/runners/flink/FlinkRunner.java | 36 +-
runners/google-cloud-dataflow-java/pom.xml | 19 +-
.../dataflow/DataflowPipelineTranslator.java | 37 +-
.../beam/runners/dataflow/DataflowRunner.java | 62 +-
.../options/DataflowPipelineWorkerPoolOptions.java | 1 -
.../beam/runners/dataflow/DataflowRunnerTest.java | 36 -
runners/java-fn-execution/pom.xml | 5 -
runners/spark/build.gradle | 5 +-
runners/spark/pom.xml | 96 +-
.../beam/runners/spark/SparkContextOptions.java | 2 +-
.../beam/runners/spark/SparkPipelineOptions.java | 14 +-
.../org/apache/beam/runners/spark/SparkRunner.java | 13 +
.../runners/spark/io/SparkUnboundedSource.java | 6 +-
.../SparkGroupAlsoByWindowViaWindowSet.java | 5 +-
.../spark/stateful/SparkTimerInternals.java | 11 +-
.../spark/translation/MultiDoFnFunction.java | 5 +-
.../spark/translation/SparkContextFactory.java | 5 +
.../SparkGroupAlsoByWindowViaOutputBufferFn.java | 5 +-
.../spark/translation/SparkProcessContext.java | 68 +-
.../spark/translation/TranslationUtils.java | 39 +-
.../runners/spark/util/GlobalWatermarkHolder.java | 24 +-
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 137 +-
.../apache/beam/sdk/io/DefaultFilenamePolicy.java | 2 +-
.../java/org/apache/beam/sdk/io/FileBasedSink.java | 52 +-
.../main/java/org/apache/beam/sdk/io/FileIO.java | 997 +++++++-
.../java/org/apache/beam/sdk/io/TFRecordIO.java | 36 +
.../main/java/org/apache/beam/sdk/io/TextIO.java | 140 +-
.../java/org/apache/beam/sdk/io/WriteFiles.java | 11 +-
.../beam/sdk/options/PipelineOptionsFactory.java | 6 +-
.../org/apache/beam/sdk/options/ValueProvider.java | 3 +-
.../apache/beam/sdk/options/ValueProviders.java | 4 +-
.../apache/beam/sdk/transforms/Requirements.java | 10 +
.../beam/sdk/transforms/SerializableFunctions.java | 20 +-
.../java/org/apache/beam/sdk/util/ShardedFile.java | 3 +-
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 275 ++-
.../org/apache/beam/sdk/io/TFRecordIOTest.java | 137 +-
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 87 +-
.../extensions/google-cloud-platform-core/pom.xml | 18 +
sdks/java/extensions/sql/build.gradle | 11 +-
sdks/java/extensions/sql/pom.xml | 1 +
.../sql/impl/planner/BeamQueryPlanner.java | 10 +
.../beam/sdk/extensions/sql/BeamSqlDslBase.java | 2 +-
.../sdk/extensions/sql/BeamSqlNonAsciiTest.java | 57 +
.../data/BeamFnDataBufferingOutboundObserver.java | 9 +
.../BeamFnDataBufferingOutboundObserverTest.java | 19 +
sdks/java/harness/pom.xml | 17 +
.../fn/harness/control/ProcessBundleHandler.java | 1 -
.../java/org/apache/beam/sdk/io/avro/AvroIOIT.java | 4 +-
.../apache/beam/sdk/io/tfrecord/TFRecordIOIT.java | 4 +-
sdks/java/io/google-cloud-platform/pom.xml | 17 +
.../beam/sdk/io/gcp/bigtable/BigtableConfig.java | 261 ++
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 478 ++--
.../sdk/io/gcp/bigquery/BigQueryHelpersTest.java | 139 ++
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 787 ++++++
.../beam/sdk/io/gcp/bigquery/BigQueryIOTest.java | 2546 +-------------------
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 1388 +++++++++++
.../sdk/io/gcp/bigquery/FakeDatasetService.java | 51 +-
.../beam/sdk/io/gcp/bigquery/FakeJobService.java | 20 +-
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 27 +-
sdks/java/io/hadoop/jdk1.8-tests/build.gradle | 1 +
sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 24 +-
.../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 +-
sdks/java/io/jdbc/pom.xml | 4 +-
.../io/kafka/serialization/InstantSerializer.java | 2 +-
sdks/java/io/kinesis/build.gradle | 6 +-
sdks/java/io/kinesis/pom.xml | 27 +-
.../org/apache/beam/sdk/io/kinesis/KinesisIO.java | 67 +-
.../apache/beam/sdk/io/kinesis/KinesisReader.java | 2 +-
.../beam/sdk/io/kinesis/AmazonKinesisMock.java | 46 +-
.../beam/sdk/io/kinesis/KinesisReaderIT.java | 86 +-
.../beam/sdk/io/kinesis/KinesisUploader.java | 28 +-
.../java/org/apache/beam/sdk/io/xml/XmlIO.java | 265 +-
.../java/org/apache/beam/sdk/io/xml/XmlSink.java | 156 --
.../java/org/apache/beam/sdk/io/xml/XmlIOTest.java | 270 +++
.../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 280 ---
.../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 87 +-
sdks/java/javadoc/pom.xml | 6 +-
.../src/main/resources/archetype-resources/pom.xml | 6 +-
.../src/main/resources/archetype-resources/pom.xml | 6 +-
sdks/java/nexmark/build.gradle | 6 +
sdks/java/nexmark/pom.xml | 29 +-
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 121 +-
.../apache/beam/sdk/nexmark/NexmarkOptions.java | 7 +
.../org/apache/beam/sdk/nexmark/model/Auction.java | 8 +-
.../org/apache/beam/sdk/nexmark/model/Person.java | 6 +-
.../beam/sdk/nexmark/model/sql/BeamRecordSize.java | 134 ++
.../beam/sdk/nexmark/model/sql/ToBeamRecord.java | 82 +
.../model/sql/adapter/ModelAdaptersMapping.java | 124 +
.../model/sql/adapter/ModelFieldsAdapter.java} | 34 +-
.../nexmark/model/sql/adapter/package-info.java} | 24 +-
.../beam/sdk/nexmark/model/sql/package-info.java} | 24 +-
.../beam/sdk/nexmark/queries/NexmarkQuery.java | 5 +-
.../sdk/nexmark/queries/sql/NexmarkSqlQuery.java | 55 +
.../beam/sdk/nexmark/queries/sql/SqlQuery0.java | 97 +
.../sdk/nexmark/queries/sql/package-info.java} | 24 +-
.../sdk/nexmark/model/sql/BeamRecordSizeTest.java | 112 +
.../sdk/nexmark/model/sql/ToBeamRecordTest.java | 109 +
.../sql/adapter/ModelAdaptersMappingTest.java | 159 ++
.../sdk/nexmark/queries/sql/SqlQuery0Test.java | 70 +
sdks/python/apache_beam/coders/coders.py | 40 +
sdks/python/apache_beam/coders/typecoders.py | 11 +-
.../apache_beam/examples/snippets/snippets_test.py | 4 +-
.../python/apache_beam/examples/wordcount_fnapi.py | 11 +-
sdks/python/apache_beam/internal/gcp/auth.py | 8 +-
sdks/python/apache_beam/io/filebasedsource.py | 2 +-
sdks/python/apache_beam/io/filesystem.py | 4 +-
sdks/python/apache_beam/io/filesystems.py | 1 +
sdks/python/apache_beam/io/gcp/gcsio.py | 99 +-
sdks/python/apache_beam/io/gcp/gcsio_test.py | 38 -
sdks/python/apache_beam/io/hadoopfilesystem.py | 285 +++
.../python/apache_beam/io/hadoopfilesystem_test.py | 472 ++++
sdks/python/apache_beam/io/iobase.py | 15 +-
sdks/python/apache_beam/io/localfilesystem_test.py | 2 +-
sdks/python/apache_beam/io/range_trackers.py | 44 +-
sdks/python/apache_beam/io/range_trackers_test.py | 37 -
sdks/python/apache_beam/io/restriction_trackers.py | 131 +
.../apache_beam/io/restriction_trackers_test.py | 159 ++
sdks/python/apache_beam/metrics/cells.py | 13 +-
sdks/python/apache_beam/metrics/execution.py | 24 +
sdks/python/apache_beam/pipeline.py | 4 +-
sdks/python/apache_beam/pipeline_test.py | 14 +-
sdks/python/apache_beam/runners/common.pxd | 34 +-
sdks/python/apache_beam/runners/common.py | 160 +-
.../runners/dataflow/dataflow_runner.py | 4 +-
.../runners/dataflow/internal/apiclient_test.py | 131 +-
.../runners/dataflow/internal/dependency.py | 60 +-
.../apache_beam/runners/direct/direct_runner.py | 25 +-
sdks/python/apache_beam/runners/direct/executor.py | 3 +
.../runners/direct/sdf_direct_runner.py | 354 +++
.../runners/direct/sdf_direct_runner_test.py | 235 ++
.../runners/direct/transform_evaluator.py | 102 +-
.../runners/portability/fn_api_runner.py | 547 +++--
.../runners/portability/fn_api_runner_test.py | 73 +-
.../runners/portability/maptask_executor_runner.py | 4 +-
.../runners/portability/universal_local_runner.py | 2 +-
sdks/python/apache_beam/runners/runner.py | 17 +-
sdks/python/apache_beam/runners/runner_test.py | 12 +-
sdks/python/apache_beam/runners/sdf_common.py | 168 ++
.../apache_beam/runners/worker/bundle_processor.py | 65 +-
.../apache_beam/runners/worker/data_plane.py | 62 +-
.../apache_beam/runners/worker/operations.py | 9 +-
.../apache_beam/runners/worker/sdk_worker.py | 175 +-
.../apache_beam/runners/worker/sdk_worker_main.py | 38 +-
.../runners/worker/sdk_worker_main_test.py | 33 +-
.../apache_beam/runners/worker/sdk_worker_test.py | 68 +-
sdks/python/apache_beam/transforms/core.py | 24 +-
sdks/python/apache_beam/transforms/trigger.py | 3 +
sdks/python/apache_beam/typehints/opcodes.py | 51 +-
.../apache_beam/typehints/trivial_inference.py | 53 +-
.../typehints/trivial_inference_test.py | 38 +-
sdks/python/apache_beam/utils/urns.py | 3 +
sdks/python/container/boot.go | 57 +-
sdks/python/container/piputil.go | 131 +
sdks/python/gen_protos.py | 3 +-
sdks/python/generate_pydoc.sh | 7 +-
sdks/python/setup.py | 15 +-
sdks/python/tox.ini | 2 +-
settings.gradle | 4 +-
179 files changed, 10518 insertions(+), 4948 deletions(-)
--
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" <co...@beam.apache.org>.