You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by ch...@apache.org on 2015/09/10 00:13:21 UTC
[50/50] incubator-apex-core git commit: Bring the master upto date
with release-3.1.0
Bring the master upto date with release-3.1.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/d7c8964b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/d7c8964b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/d7c8964b
Branch: refs/heads/master
Commit: d7c8964b7983fc3a8df38fbc85ec5291449d8960
Parents: b7475a0 03f612f
Author: Chetan Narsude <ch...@datatorrent.com>
Authored: Wed Sep 9 15:03:26 2015 -0700
Committer: Chetan Narsude <ch...@datatorrent.com>
Committed: Wed Sep 9 15:11:34 2015 -0700
----------------------------------------------------------------------
CHANGELOG.md | 115 +++++
RELEASE.md | 71 +++
apex-app-archetype/README.md | 2 +-
apex-app-archetype/pom.xml | 2 +-
.../projects/basic/archetype.properties | 2 +-
apex-conf-archetype/README.md | 2 +-
apex-conf-archetype/pom.xml | 2 +-
.../projects/basic/archetype.properties | 2 +-
api/pom.xml | 44 +-
.../java/com/datatorrent/api/AutoMetric.java | 11 +-
.../main/java/com/datatorrent/api/Context.java | 7 +
.../annotation/InputPortFieldAnnotation.java | 10 +-
.../annotation/OutputPortFieldAnnotation.java | 10 +
bufferserver/pom.xml | 4 +-
.../datatorrent/bufferserver/server/Server.java | 2 +-
.../datatorrent/bufferserver/util/System.java | 2 +-
.../bufferserver/client/SubscriberTest.java | 4 +-
.../bufferserver/server/ServerTest.java | 4 +-
.../bufferserver/storage/DiskStorageTest.java | 9 +-
common/pom.xml | 48 +-
.../common/util/AsyncFSStorageAgent.java | 143 ++++++
.../datatorrent/common/util/BaseOperator.java | 2 +
.../datatorrent/common/util/FSStorageAgent.java | 5 +-
.../common/codec/JsonStreamCodecTest.java | 15 +-
.../common/util/AsyncFSStorageAgentTest.java | 133 ++++++
engine/pom.xml | 12 +-
.../java/com/datatorrent/stram/StramClient.java | 8 +-
.../datatorrent/stram/StramLocalCluster.java | 4 +-
.../stram/StreamingAppMasterService.java | 2 +-
.../stram/StreamingContainerManager.java | 170 +++++---
.../java/com/datatorrent/stram/cli/DTCli.java | 34 +-
.../java/com/datatorrent/stram/engine/Node.java | 64 ++-
.../stram/engine/StreamingContainer.java | 8 +-
.../stram/plan/logical/LogicalPlan.java | 30 +-
.../plan/logical/LogicalPlanConfiguration.java | 22 +-
.../stram/plan/physical/PhysicalPlan.java | 40 +-
.../datatorrent/stram/stream/FastPublisher.java | 2 +
.../datatorrent/stram/util/FSJsonLineFile.java | 24 +-
.../stram/webapp/OperatorDiscoverer.java | 434 +++++++++++--------
.../stram/webapp/StramWebServices.java | 8 +-
.../com/datatorrent/stram/webapp/TypeGraph.java | 119 ++++-
.../com/datatorrent/stram/CheckpointTest.java | 11 +-
.../stram/LogicalPlanModificationTest.java | 22 +-
.../com/datatorrent/stram/MockContainer.java | 2 +-
.../com/datatorrent/stram/PartitioningTest.java | 26 +-
.../stram/StramLocalClusterTest.java | 22 +-
.../datatorrent/stram/StramMiniClusterTest.java | 9 +-
.../datatorrent/stram/StramRecoveryTest.java | 58 ++-
.../com/datatorrent/stram/StreamCodecTest.java | 80 +---
.../stram/StreamingContainerManagerTest.java | 139 +++++-
.../stram/client/AppPackageTest.java | 2 +-
.../stram/debug/TupleRecorderTest.java | 3 +
.../stram/engine/AtLeastOnceTest.java | 16 +
.../stram/engine/AtMostOnceTest.java | 2 +-
.../stram/engine/AutoMetricTest.java | 12 +-
.../stram/engine/InputOperatorTest.java | 5 +-
.../stram/engine/ProcessingModeTests.java | 11 +-
.../stram/engine/RecoverableInputOperator.java | 10 +-
.../datatorrent/stram/engine/SliderTest.java | 5 +
.../com/datatorrent/stram/engine/StatsTest.java | 10 +-
.../stram/engine/StreamingContainerTest.java | 6 +
.../stram/engine/WindowGeneratorTest.java | 11 +-
.../plan/LogicalPlanConfigurationTest.java | 83 +++-
.../datatorrent/stram/plan/LogicalPlanTest.java | 16 +-
.../stram/plan/SchemaTestOperator.java | 33 ++
.../stram/plan/physical/PhysicalPlanTest.java | 9 +-
.../stram/stream/FastStreamTest.java | 6 +-
.../stram/stream/OiOEndWindowTest.java | 5 +
.../stram/stream/SocketStreamTest.java | 8 +-
.../stram/support/StramTestSupport.java | 7 +-
.../stram/webapp/OperatorDiscoveryTest.java | 163 +++++--
.../stram/webapp/StramWebServicesTest.java | 6 +-
.../src/test/resources/schemaTestTopology.json | 43 ++
.../resources/testAppPackage/mydtapp/pom.xml | 2 +-
pom.xml | 2 +-
75 files changed, 1886 insertions(+), 591 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d7c8964b/CHANGELOG.md
----------------------------------------------------------------------
diff --cc CHANGELOG.md
index 141ff79,0158919..0f06d2d
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@@ -1,11 -1,27 +1,126 @@@
Apex Changelog
========================================================================================================================
+Version 3.0.0
+------------------------------------------------------------------------------------------------------------------------
+
+* Add jersey client jar for app execution
+* Must catch NoClassDefFoundError when processing operator classes in jar, previously catching Throwable was changed to catching Exception
++* Do not catch throwable in DTCli and exit immediately when worker thread throws an error
++* Depend on published netlet version
++* Catch throwable when executing command because launching an app can throw java.lang.VerifyError: bad type on operand stack
++* Removed runOnlyOnce when generating dt-git.properties and generate even when not using release profile
++* Undeploy heartbeat requests are not processes if container is idle
++* Fix potential NPE
++* Comment hide the actually type for string types(URI, URL, Class etc) and add 2 missing wrapper types
++* Fixed typo in webservice url for get-physical-plan command
++* Resolve deleting checkpoint in different thread
++* Removed duplicate code and added unit test for json stream codec
++* APEX-11 #resolve added checkpoint metric
++* Have default connect and read timeouts because the jersey defaults are infinity and that blocks threads and those threads can't be interrupted either
++* Removed invalid app data push annotation
++* Use FileContext instead of FileSystem to write the meta json file
++* Comment added required memory and required vcores in the appinfo
++* Comment filter abstract types from getAssignableClasses call and rename initializable to instantiable
++* Deploy all artifacts by default.
++* Comment fix the bug in trimming the graph
++* HA support for stram webservice filter.
++* Removed dependencies in filter to hadoop classes with private audience as their interface has changed from Hadoop 2.2 to 2.6
++* Related doc updates
++* Comment Prune the external dependencies from type graph and break the type circle
++* Fixing class not found issue due to missing dt-common jar.
++* Resolve removed old alert support in stram
++* Use tokens from Credentials as UserGroupInformation.getTokens() returns HDFS Namenode hidden tokens that should not be passed to containers.
++* Support for RM delegation token renewal in secure HA environments
++* Resolve fixed bug when custom metric keys are not stored correctly in meta file
++* Comment Use apache xbean shaded asm to resolve jdk 8 class and avoid conflict with old asm version that hadoop depends on
++* APEX-5 #resolve Set APPLICATION_PATH for unit tests that launch an app
++* Resolve Added dependency on dt-engine in test scope of archetype
++* Corrected the place to get pom.properties for version info
++* Token authentication support for buffer server
++* Adding default aggregator for primitive customMetrics
++* Netlet has a few fixed related to non firing OP_CONNECT call, so test against those.
++* Print stack trace when exception occurs
++* Commented out app master mb in properties.xml
++* Ability to extract javadocs as xml
++* Moved PAYLOAD_VALUE out of default case. Count number of skipped payload tuples.
++* Resolve fixing ASM signature visitor to add upper bound to Object for unbounded wild card entries
++* Removing unused imports
++* Addressing review comments
++* Reverting code format change
++* Resolve Added END_STREAM to be distributed to physical nodes. Introduced debug message for default case.
++* Comment removed the coding convention
++* Resolve Changed reading of portTypeInfo, port annotations in operator discoverer to read from ASM
++* Added test case for operator discoverer
++* Updated Type Discovery tests to getPortInfo via ASM instead of reflection API
++* Addressing review comments
++* Removed condition for public, final, transient ports when generating typeGraph using ASM
++* Added a class for storing input, output port annotations
++* Added port type info handling for ports other than DefaultInputPort and DefaultOutputPort types
++* Updated a test case for the same
++* Adding annotations to list.. Missed in previous commit
++* Added rescan of typegraph to capture operator ports
++* Changing ASMUtil utility methods to public
++* Addressing review comments.
++* Correcting jar entry name
++* Updated a test after merge
++* Removed system.out print from test cases and added debug print on failure
++* Removing temp file
++* Reversed the order of setting jvm_options for app master
++* Resolve Added validation for root operator should be input operator in logical plan validation
++* Fixing formatting and added lincense header
++* Removed extra line spaces
++* Correcting output port in dt-site.xml stream connection
++* Resolve Added validation for root operator should be input operator in logical plan validation
++* Fixing formatting and added lincense header
++* Removed extra line spaces
++* Correcting output port in dt-site.xml stream connection
++* Removing white spaces
++* Renaming the properties to reflect the code change
++* Comment add the missing open tag <p>
++* Removed unnecessary depedency
++* Resolve Remove unnecessary unpack in app package archetype
++* Resolve support java.lang.Class and resolve uiType to special types
++* Resolve update dependency to dt-common in archetype
++* Resolve added the container jvm options for app master
++* Resolve attach apa file as an artifact
++* Resolve test app package should depend on dt-common instead of dt-engine
++* Removed additional license header
++* Removed incorrectly left in license headers due to incorrect formatting
++* Put back revision info.
++* Added a TODO note for handling of stram delegation tokens for future
++* Comment Add dag attributes to LogicalPlanSerializer
++* Using resolveType method in OperatorDiscoverer to describe an attribute completely
++* Switch to Java7 and update compiler plugin.
++* Remove invalid module references.
++* Separated out HA token creation from non-HA case as it involves special handling that is subject to change if Hadoop's internals change.
++* Resolve - Add all resource managers to RM delegation token service.
++* Fix CLI script.
++* Stram directory is moved to engine
++* Fix depenency plugin version (2.3 shipped with maven 3.2.5 pulls dependencies from test scope)
++* Setup the pom files for OS with optimized dependencies
++* Changed the license header to Apache 2.0 license.
+
+ Version 3.1.0
+ ------------------------------------------------------------------------------------------------------------------------
+
+ ** Improvement
+ * [APEX-12] - Fix Base Operator To Not Show Name Property In App Builder
+
+ ** Bug
+ * [APEX-35] - Test exceptions due to missing directory in saveMetaInfo
+ * [APEX-36] - FSStorageAgent to account for HDFS lease when writing checkpoint files
+ * [APEX-37] - Container and operator json line file in StreamingContainerManager should not be appended from previous app attempt
+ * [APEX-43] - SchemaSupport: TUPLE_CLASS attribute should use Class2String StringCodec
+ * [APEX-56] - Controlled plan modification on operator shutdown
+
+
+
+ Version 3.0.0
+ ------------------------------------------------------------------------------------------------------------------------
+
+ * Add jersey client jar for app execution
+ * Must catch NoClassDefFoundError when processing operator classes in jar, previously catching Throwable was changed to catching Exception
* Do not catch throwable in DTCli and exit immediately when worker thread throws an error
* Depend on published netlet version
* Catch throwable when executing command because launching an app can throw java.lang.VerifyError: bad type on operand stack
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d7c8964b/engine/src/main/java/com/datatorrent/stram/StreamingAppMasterService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d7c8964b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
----------------------------------------------------------------------
diff --cc engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
index 95f4648,7002c1d..a9502b7
--- a/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
+++ b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java
@@@ -494,12 -496,10 +496,13 @@@ public class StreamingContainerManager
}
IOUtils.closeQuietly(containerFile);
- for (FSJsonLineFile operatorFile : operatorFiles.values()) {
- IOUtils.closeQuietly(operatorFile);
+ IOUtils.closeQuietly(operatorFile);
+ if(poolExecutor != null) {
+ poolExecutor.shutdown();
}
+ if(poolExecutor != null) {
+ poolExecutor.shutdown();
+ }
}
public void subscribeToEvents(Object listener)
@@@ -2246,11 -2265,11 +2268,15 @@@
oi.currentWindowId = toWsWindowId(os.currentWindowId.get());
if (os.lastHeartbeat != null) {
oi.lastHeartbeat = os.lastHeartbeat.getGeneratedTms();
+ }
+ if (os.checkpointStats != null) {
+ oi.checkpointTime = os.checkpointStats.checkpointTime;
+ oi.checkpointStartTime = os.checkpointStats.checkpointStartTime;
}
+ if (os.checkpointStats != null) {
+ oi.checkpointTime = os.checkpointStats.checkpointTime;
+ oi.checkpointStartTime = os.checkpointStats.checkpointStartTime;
+ }
oi.checkpointTimeMA = os.checkpointTimeMA.getAvg();
for (PortStatus ps : os.inputPortStatusList.values()) {
PortInfo pinfo = new PortInfo();