You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2021/09/22 15:24:45 UTC
[incubator-wayang] branch WAYANG-8 created (now 42e8ece)
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a change to branch WAYANG-8
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git.
at 42e8ece Merge pull request #11 from ro-pardo/python-api
This branch includes the following new commits:
new 828c9c2 Merge pull request #9 from ro-pardo/python-api
new 42e8ece Merge pull request #11 from ro-pardo/python-api
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-wayang] 02/02: Merge pull request #11 from
ro-pardo/python-api
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch WAYANG-8
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 42e8ece561f253710591bee2288ba4ba569d41a3
Merge: 828c9c2 9c15cdd
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Tue Mar 23 21:30:14 2021 -0300
Merge pull request #11 from ro-pardo/python-api
Python api merge
protobuf/pywayangplan.proto | 68 -----
pywayang/config/pywayang_config.ini | 2 +-
pywayang/graph/__init__.py | 3 +
pywayang/graph/graph.py | 13 +-
pywayang/graph/node.py | 4 +
pywayang/graph/traversal.py | 6 +-
pywayang/graph/visitant.py | 11 +-
pywayang/orchestrator/__init__.py | 1 +
pywayang/orchestrator/dataquanta.py | 129 +++++++++-
pywayang/orchestrator/main.py | 86 +++++--
pywayang/orchestrator/operator.py | 35 +--
pywayang/orchestrator/plan.py | 5 +-
pywayang/protobuf/planwriter.py | 284 +++++++++++++--------
.../wayang-api-python/src/main/python/worker.py | 16 --
.../api/rest/server/spring/decoder/pyplanreader.sh | 20 --
.../server/spring/general/WayangController.java | 212 ++++++++++-----
.../src/main/proto/pywayangplan.proto | 35 +--
.../src/main/python}/pyplangenerator.sh | 0
18 files changed, 572 insertions(+), 358 deletions(-)
[incubator-wayang] 01/02: Merge pull request #9 from
ro-pardo/python-api
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch WAYANG-8
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 828c9c235dd7376b7dd937bc1c70baec79eab47b
Merge: ff0cb9f 24a55f8
Author: berttty <be...@scalytics.io>
AuthorDate: Fri Mar 5 13:30:01 2021 -0300
Merge pull request #9 from ro-pardo/python-api
Python api
.travis.yml | 35 +++-
README.md | 6 +-
pom.xml | 47 ++----
.../pyplangenerator.sh | 4 +-
.../Action.java => protobuf/pywayangplan.proto | 71 +++++---
.../config/__init__.py | 4 +-
pywayang/config/config_reader.py | 51 ++++++
.../config/pywayang_config.ini | 22 ++-
.../graph/__init__.py | 4 +-
pywayang/graph/graph.py | 68 ++++++++
pywayang/graph/node.py | 44 +++++
pywayang/graph/traversal.py | 49 ++++++
pywayang/graph/visitant.py | 51 ++++++
.../orchestrator/__init__.py | 5 +-
pywayang/orchestrator/dataquanta.py | 136 +++++++++++++++
pywayang/orchestrator/main.py | 76 +++++++++
pywayang/orchestrator/operator.py | 110 ++++++++++++
.../orchestrator/plan.py | 22 ++-
.../protobuf/__init__.py | 2 +-
pywayang/protobuf/planwriter.py | 152 +++++++++++++++++
wayang-api/pom.xml | 1 +
wayang-api/wayang-api-python/pom.xml | 46 ++++-
.../wayang/api/python/executor/ProcessFeeder.java | 177 ++++++++++++++++++++
.../api/python/executor/ProcessReceiver.java | 52 ++++++
.../api/python/executor/PythonProcessCaller.java | 118 +++++++++++++
.../api/python/executor/PythonWorkerManager.java | 64 +++++++
.../wayang/api/python/executor/ReaderIterator.java | 86 ++++++++++
.../wayang/api/python/function/PythonUdf.java | 19 +--
.../api/python/function/WrappedPythonFunction.java | 43 +++++
.../wayang-api-python/src/main/python/worker.py | 186 +++++++++++++++++++++
.../wayang-api-python-defaults.properties | 3 +-
.../org/apache/wayang/api/python/BasicPython.java | 67 ++++++++
.../org/apache/wayang/api/python/SortPython.java | 102 +++++++++++
wayang-api/wayang-api-rest/pom.xml | 97 +++++++++++
.../org/apache/wayang/api/rest/WayangRESTAPI.java | 20 +--
.../api/rest/server/spring/WayangApplication.java | 22 +--
.../rest/server/spring/decoder/pyplanreader.sh} | 4 +-
.../server/spring/general/WayangController.java | 159 ++++++++++++++++++
.../src/main/resources/application.properties} | 2 +-
wayang-commons/pom.xml | 13 +-
.../wayang/basic/operators/SampleOperator.java | 6 +-
.../wayang/basic/operators/TextFileSource.java | 6 +-
.../wayang/basic/operators/TextFileSourceTest.java | 7 +-
wayang-commons/wayang-core/pom.xml | 8 +
.../org/apache/wayang/core/api/Configuration.java | 6 +-
.../main/java/org/apache/wayang/core/api/Job.java | 6 +-
.../org/apache/wayang/core/api/WayangContext.java | 6 +-
.../core/api/configuration/CollectionProvider.java | 6 +-
.../configuration/ExplicitCollectionProvider.java | 6 +-
.../core/api/configuration/KeyValueProvider.java | 6 +-
.../core/api/configuration/ValueProvider.java | 6 +-
.../wayang/core/mapping/PlanTransformation.java | 6 +-
.../wayang/core/optimizer/OptimizationContext.java | 6 +-
.../wayang/core/optimizer/SanityChecker.java | 6 +-
.../cardinality/CardinalityEstimatorManager.java | 6 +-
.../optimizer/cardinality/CardinalityPusher.java | 6 +-
.../SwitchForwardCardinalityEstimator.java | 4 +-
.../optimizer/channels/ChannelConversionGraph.java | 6 +-
.../optimizer/costs/LoadProfileEstimators.java | 6 +-
.../optimizer/enumeration/ExecutionTaskFlow.java | 6 +-
.../enumeration/ExecutionTaskFlowCompiler.java | 4 +-
.../enumeration/LatentOperatorPruningStrategy.java | 10 +-
.../core/optimizer/enumeration/LoopEnumerator.java | 4 +-
.../optimizer/enumeration/PlanEnumeration.java | 6 +-
.../core/optimizer/enumeration/PlanEnumerator.java | 6 +-
.../optimizer/enumeration/PlanImplementation.java | 6 +-
.../enumeration/StageAssignmentTraversal.java | 6 +-
.../wayang/core/plan/executionplan/Channel.java | 8 +-
.../core/plan/executionplan/ExecutionPlan.java | 6 +-
.../core/plan/wayangplan/ExecutionOperator.java | 4 +-
.../wayang/core/plan/wayangplan/LoopIsolator.java | 4 +-
.../core/plan/wayangplan/OperatorContainer.java | 4 +-
.../wayang/core/plan/wayangplan/PlanTraversal.java | 6 +-
.../wayang/core/plan/wayangplan/SlotMapping.java | 6 +-
.../wayang/core/plan/wayangplan/WayangPlan.java | 6 +-
.../traversal/AbstractTopologicalTraversal.java | 6 +-
.../core/platform/AbstractChannelInstance.java | 4 +-
.../core/platform/CardinalityBreakpoint.java | 6 +-
.../core/platform/CrossPlatformExecutor.java | 6 +-
.../wayang/core/platform/ExecutorTemplate.java | 6 +-
.../org/apache/wayang/core/platform/Junction.java | 6 +-
.../platform/lineage/LazyExecutionLineageNode.java | 4 +-
.../core/profiling/CardinalityRepository.java | 6 +-
.../apache/wayang/core/profiling/ExecutionLog.java | 6 +-
.../core/util/AbstractReferenceCountable.java | 7 +-
.../java/org/apache/wayang/core/util/Action.java | 4 +-
.../org/apache/wayang/core/util/JuelUtils.java | 6 +-
.../java/org/apache/wayang/core/util/Logging.java | 6 +-
.../apache/wayang/core/util/ReflectionUtils.java | 6 +-
.../apache/wayang/core/util/fs/FileSystems.java | 8 +-
.../wayang/core/util/fs/HadoopFileSystem.java | 10 +-
.../wayang/core/util/fs/LocalFileSystem.java | 6 +-
.../core/util/ConsumerIteratorAdapterTest.java | 6 +-
wayang-commons/wayang-serializable/pom.xml | 80 +++++++++
.../src/main/proto/pywayangplan.proto | 73 ++++++++
.../flink/compiler/WayangFileOutputFormat.java | 6 +-
.../wayang/flink/platform/FlinkPlatform.java | 6 +-
.../wayang/giraph/Algorithm/PageRankAlgorithm.java | 7 +-
.../giraph/operators/GiraphPageRankOperator.java | 6 +-
.../operators/GraphChiPageRankOperator.java | 6 +-
.../wayang/java/operators/JavaObjectFileSink.java | 4 +-
.../java/operators/JavaObjectFileSource.java | 4 +-
wayang-platforms/wayang-jdbc-template/pom.xml | 17 ++
.../apache/wayang/jdbc/execution/JdbcExecutor.java | 6 +-
.../wayang/jdbc/operators/JdbcTableSource.java | 4 +-
.../wayang/jdbc/operators/SqlToStreamOperator.java | 4 +-
.../spark/operators/SparkObjectFileSink.java | 4 +-
.../spark/operators/SparkObjectFileSource.java | 6 +-
.../wayang/spark/platform/SparkPlatform.java | 6 +-
.../wayang/profiler/java/OperatorProfiler.java | 6 +-
.../wayang/profiler/java/SourceProfiler.java | 5 +-
.../profiler/log/DynamicLoadProfileEstimators.java | 6 +-
.../wayang/profiler/log/GeneticOptimizerApp.java | 6 +-
.../apache/wayang/profiler/log/LogEvaluator.java | 6 +-
.../profiler/spark/SparkOperatorProfiler.java | 6 +-
.../wayang/profiler/util/ProfilingUtils.java | 4 +-
.../apache/wayang/profiler/util/RrdAccessor.java | 6 +-
{wayang-api => wayang-resources}/pom.xml | 39 +++--
.../src/main/assembly/resources.xml | 39 ++---
.../src/main/resources/assemblies/resources.xml | 39 ++---
.../apache/wayang/tests/GraphChiIntegrationIT.java | 12 +-
121 files changed, 2557 insertions(+), 412 deletions(-)