You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by sa...@apache.org on 2016/05/13 23:09:24 UTC
[22/27] incubator-apex-site git commit: Adding a link to apex-3.2
docs instead of copy of apex-3.3 docs
http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/1052900a/docs/apex-3.2/application_packages/index.html
----------------------------------------------------------------------
diff --git a/docs/apex-3.2/application_packages/index.html b/docs/apex-3.2/application_packages/index.html
deleted file mode 100644
index faae829..0000000
--- a/docs/apex-3.2/application_packages/index.html
+++ /dev/null
@@ -1,793 +0,0 @@
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-
-
- <title>Packages - Apache Apex Documentation</title>
-
-
- <link rel="shortcut icon" href="../favicon.ico">
-
-
-
- <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
-
- <link rel="stylesheet" href="../css/theme.css" type="text/css" />
- <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
- <link rel="stylesheet" href="../css/highlight.css">
-
-
- <script>
- // Current page data
- var mkdocs_page_name = "Packages";
- var mkdocs_page_input_path = "application_packages.md";
- var mkdocs_page_url = "/application_packages/";
- </script>
-
- <script src="../js/jquery-2.1.1.min.js"></script>
- <script src="../js/modernizr-2.8.3.min.js"></script>
- <script type="text/javascript" src="../js/highlight.pack.js"></script>
- <script src="../js/theme.js"></script>
-
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
- <div class="wy-side-nav-search">
- <a href=".." class="icon icon-home"> Apache Apex Documentation</a>
- <div role="search">
- <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- </form>
-</div>
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
- <ul class="current">
-
- <li>
- <li class="toctree-l1 ">
- <a class="" href="..">Apache Apex</a>
-
- </li>
-<li>
-
- <li>
- <ul class="subnav">
- <li><span>Development</span></li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../apex_development_setup/">Development Setup</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../application_development/">Applications</a>
-
- </li>
-
-
-
- <li class="toctree-l1 current">
- <a class="current" href="./">Packages</a>
-
- <ul>
-
- <li class="toctree-l3"><a href="#apache-apex-packages">Apache Apex Packages</a></li>
-
-
- <li class="toctree-l3"><a href="#application-packages">Application Packages</a></li>
-
- <li><a class="toctree-l4" href="#requirements">Requirements</a></li>
-
- <li><a class="toctree-l4" href="#creating-your-first-apex-app-package">Creating Your First Apex App Package</a></li>
-
- <li><a class="toctree-l4" href="#writing-your-own-app-package">Writing Your Own App Package</a></li>
-
- <li><a class="toctree-l4" href="#zip-structure-of-application-package">Zip Structure of Application Package</a></li>
-
- <li><a class="toctree-l4" href="#examining-and-launching-application-packages-through-cli">Examining and Launching Application Packages Through CLI</a></li>
-
-
- <li class="toctree-l3"><a href="#configuration-packages">Configuration Packages</a></li>
-
- <li><a class="toctree-l4" href="#creating-configuration-packages">Creating Configuration Packages</a></li>
-
- <li><a class="toctree-l4" href="#assembling-your-own-configuration-package">Assembling your own configuration package</a></li>
-
-
- </ul>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../operator_development/">Operators</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../autometrics/">AutoMetric API</a>
-
- </li>
-
-
- </ul>
-<li>
-
- <li>
- <ul class="subnav">
- <li><span>Operations</span></li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../dtcli/">dtCli</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../security/">Security</a>
-
- </li>
-
-
- </ul>
-<li>
-
- <li>
- <li class="toctree-l1 ">
- <a class="" href="../compatibility/">Compatibility</a>
-
- </li>
-<li>
-
- </ul>
- </div>
-
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="..">Apache Apex Documentation</a>
- </nav>
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
- <div role="navigation" aria-label="breadcrumbs navigation">
- <ul class="wy-breadcrumbs">
- <li><a href="..">Docs</a> »</li>
-
-
-
- <li>Development »</li>
-
-
-
- <li>Packages</li>
- <li class="wy-breadcrumbs-aside">
-
- </li>
- </ul>
- <hr/>
-</div>
- <div role="main">
- <div class="section">
-
- <h1 id="apache-apex-packages">Apache Apex Packages</h1>
-<h1 id="application-packages">Application Packages</h1>
-<p>An Apache Apex Application Package is a zip file that contains all the
-necessary files to launch an application in Apache Apex. It is the
-standard way for assembling and sharing an Apache Apex application.</p>
-<h2 id="requirements">Requirements</h2>
-<p>You will need have the following installed:</p>
-<ol>
-<li>Apache Maven 3.0 or later (for assembling the App Package)</li>
-<li>Apache Apex 3.2.0 or later (for launching the App Package in your cluster)</li>
-</ol>
-<h2 id="creating-your-first-apex-app-package">Creating Your First Apex App Package</h2>
-<p>You can create an Apex Application Package using your Linux command
-line, or using your favorite IDE.</p>
-<h3 id="using-command-line">Using Command Line</h3>
-<p>First, change to the directory where you put your projects, and create
-an Apex application project using Maven by running the following
-command. Replace "com.example", "mydtapp" and "1.0-SNAPSHOT" with the
-appropriate values (make sure this is all on one line):</p>
-<pre><code>$ mvn archetype:generate \
- -DarchetypeGroupId=org.apache.apex \
- -DarchetypeArtifactId=apex-app-archetype -DarchetypeVersion=3.2.0-incubating \
- -DgroupId=com.example -Dpackage=com.example.mydtapp -DartifactId=mydtapp \
- -Dversion=1.0-SNAPSHOT
-</code></pre>
-<p>This creates a Maven project named "mydtapp". Open it with your favorite
-IDE (e.g. NetBeans, Eclipse, IntelliJ IDEA). In the project, there is a
-sample DAG that generates a number of tuples with a random number and
-prints out "hello world" and the random number in the tuples. The code
-that builds the DAG is in
-src/main/java/com/example/mydtapp/Application.java, and the code that
-runs the unit test for the DAG is in
-src/test/java/com/example/mydtapp/ApplicationTest.java. Try it out by
-running the following command:</p>
-<pre><code>$cd mydtapp; mvn package
-</code></pre>
-<p>This builds the App Package runs the unit test of the DAG. You should
-be getting test output similar to this:</p>
-<pre><code> -------------------------------------------------------
- TESTS
- -------------------------------------------------------
-
- Running com.example.mydtapp.ApplicationTest
- hello world: 0.8015370953286478
- hello world: 0.9785359225545481
- hello world: 0.6322611586644047
- hello world: 0.8460953663451775
- hello world: 0.5719372906929072
- hello world: 0.6361174312337172
- hello world: 0.14873007534816318
- hello world: 0.8866986277418261
- hello world: 0.6346526809866057
- hello world: 0.48587295703904465
- hello world: 0.6436832429676687
-
- ...
-
- Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 11.863
- sec
-
- Results :
-
- Tests run: 1, Failures: 0, Errors: 0, Skipped: 0
-</code></pre>
-
-<p>The "mvn package" command creates the App Package file in target
-directory as target/mydtapp-1.0-SNAPSHOT.apa. You will be able to use
-that App Package file to launch this sample application in your actual
-Apex installation.</p>
-<p>Alternatively you can perform the same steps within your IDE (IDEA IntelliJ, Eclipse, NetBeans all support it). Please check the IDE documentation for details.</p>
-<p>Group ID: org.apache.apex
-Artifact ID: apex-app-archetype
-Version: 3.2.0-incubating (or any later version)</p>
-<h2 id="writing-your-own-app-package">Writing Your Own App Package</h2>
-<p>Please refer to the [Application Developer Guide][application_development.md] on the basics on how to write an Apache Apex application. In your AppPackage project, you can add custom operators (refer to <a href="../operator_development/">Operator Development Guide</a>, project dependencies, default and required configuration properties, pre-set configurations and other metadata. Note that you can also specify the DAG using Java, JSON or properties files. </p>
-<h3 id="adding-and-removing-project-dependencies">Adding (and removing) project dependencies</h3>
-<p>Under the project, you can add project dependencies in pom.xml, or do it
-through your IDE. Here\u2019s the section that describes the dependencies in
-the default pom.xml:</p>
-<pre><code> <dependencies>
- <!-- add your dependencies here -->
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>${apex.version}</version>
- <!--
- If you know your application do not need the transitive dependencies that are pulled in by malhar-library,
- Uncomment the following to reduce the size of your app package.
- -->
- <!--
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- -->
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-</code></pre>
-
-<p>By default, as shown above, the default dependencies include
-malhar-library in compile scope, dt-engine in provided scope, and junit
-in test scope. Do not remove these three dependencies since they are
-necessary for any Apex application. You can, however, exclude
-transitive dependencies from malhar-library to reduce the size of your
-App Package, provided that none of the operators in malhar-library that
-need the transitive dependencies will be used in your application.</p>
-<p>In the sample application, it is safe to remove the transitive
-dependencies from malhar-library, by uncommenting the "exclusions"
-section. It will reduce the size of the sample App Package from 8MB to
-700KB.</p>
-<p>Note that if we exclude *, in some versions of Maven, you may get
-warnings similar to the following:</p>
-<pre><code>
- [WARNING] 'dependencies.dependency.exclusions.exclusion.groupId' for
- org.apache.apex:malhar-library:jar with value '*' does not match a
- valid id pattern.
-
- [WARNING]
- [WARNING] It is highly recommended to fix these problems because they
- threaten the stability of your build.
- [WARNING]
- [WARNING] For this reason, future Maven versions might no longer support
- building such malformed projects.
- [WARNING]
-
-</code></pre>
-
-<p>This is a bug in early versions of Maven 3. The dependency exclusion is
-still valid and it is safe to ignore these warnings.</p>
-<h3 id="application-configuration">Application Configuration</h3>
-<p>A configuration file can be used to configure an application. Different
-kinds of configuration parameters can be specified. They are application
-attributes, operator attributes and properties, port attributes, stream
-properties and application specific properties. They are all specified
-as name value pairs, in XML format, like the following.</p>
-<pre><code><?xml version="1.0"?>
-<configuration>
- <property>
- <name>some_name_1</name>
- <value>some_default_value</value>
- </property>
- <property>
- <name>some_name_2</name>
- <value>some_default_value</value>
- </property>
-</configuration>
-</code></pre>
-
-<h3 id="application-attributes">Application attributes</h3>
-<p>Application attributes are used to specify the platform behavior for the
-application. They can be specified using the parameter
-<code>dt.attr.<attribute></code>. The prefix \u201cdt\u201d is a constant, \u201cattr\u201d is a
-constant denoting an attribute is being specified and <code><attribute></code>
-specifies the name of the attribute. Below is an example snippet setting
-the streaming windows size of the application to be 1000 milliseconds.</p>
-<pre><code> <property>
- <name>dt.attr.STREAMING_WINDOW_SIZE_MILLIS</name>
- <value>1000</value>
- </property>
-</code></pre>
-
-<p>The name tag specifies the attribute and value tag specifies the
-attribute value. The name of the attribute is a JAVA constant name
-identifying the attribute. The constants are defined in
-com.datatorrent.api.Context.DAGContext and the different attributes can
-be specified in the format described above.</p>
-<h3 id="operator-attributes">Operator attributes</h3>
-<p>Operator attributes are used to specify the platform behavior for the
-operator. They can be specified using the parameter
-<code>dt.operator.<operator-name>.attr.<attribute></code>. The prefix \u201cdt\u201d is a
-constant, \u201coperator\u201d is a constant denoting that an operator is being
-specified, <code><operator-name></code> denotes the name of the operator, \u201cattr\u201d is
-the constant denoting that an attribute is being specified and
-<code><attribute></code> is the name of the attribute. The operator name is the
-same name that is specified when the operator is added to the DAG using
-the addOperator method. An example illustrating the specification is
-shown below. It specifies the number of streaming windows for one
-application window of an operator named \u201cinput\u201d to be 10</p>
-<pre><code><property>
- <name>dt.operator.input.attr.APPLICATION_WINDOW_COUNT</name>
- <value>10</value>
-</property>
-</code></pre>
-
-<p>The name tag specifies the attribute and value tag specifies the
-attribute value. The name of the attribute is a JAVA constant name
-identifying the attribute. The constants are defined in
-com.datatorrent.api.Context.OperatorContext and the different attributes
-can be specified in the format described above.</p>
-<h3 id="operator-properties">Operator properties</h3>
-<p>Operators can be configured using operator specific properties. The
-properties can be specified using the parameter
-<code>dt.operator.<operator-name>.prop.<property-name></code>. The difference
-between this and the operator attribute specification described above is
-that the keyword \u201cprop\u201d is used to denote that it is a property and
-<code><property-name></code> specifies the property name. An example illustrating
-this is specified below. It specifies the property \u201chostname\u201d of the
-redis server for a \u201credis\u201d output operator.</p>
-<pre><code> <property>
- <name>dt.operator.redis.prop.host</name>
- <value>127.0.0.1</value>
- </property>
-</code></pre>
-
-<p>The name tag specifies the property and the value specifies the property
-value. The property name is converted to a setter method which is called
-on the actual operator. The method name is composed by appending the
-word \u201cset\u201d and the property name with the first character of the name
-capitalized. In the above example the setter method would become
-setHost. The method is called using JAVA reflection and the property
-value is passed as an argument. In the above example the method setHost
-will be called on the \u201credis\u201d operator with \u201c127.0.0.1\u201d as the argument.</p>
-<h3 id="port-attributes">Port attributes</h3>
-<p>Port attributes are used to specify the platform behavior for input and
-output ports. They can be specified using the parameter <code>dt.operator.<operator-name>.inputport.<port-name>.attr.<attribute></code>
-for input port and <code>dt.operator.<operator-name>.outputport.<port-name>.attr.<attribute></code>
-for output port. The keyword \u201cinputport\u201d is used to denote an input port
-and \u201coutputport\u201d to denote an output port. The rest of the specification
-follows the conventions described in other specifications above. An
-example illustrating this is specified below. It specifies the queue
-capacity for an input port named \u201cinput\u201d of an operator named \u201crange\u201d to
-be 4k.</p>
-<pre><code><property>
- <name>dt.operator.range.inputport.input.attr.QUEUE_CAPACITY</name>
- <value>4000</value>
-</property>
-</code></pre>
-
-<p>The name tag specifies the attribute and value tag specifies the
-attribute value. The name of the attribute is a JAVA constant name
-identifying the attribute. The constants are defined in
-com.datatorrent.api.Context.PortContext and the different attributes can
-be specified in the format described above.</p>
-<p>The attributes for an output port can also be specified in a similar way
-as described above with a change that keyword \u201coutputport\u201d is used
-instead of \u201cintputport\u201d. A generic keyword \u201cport\u201d can be used to specify
-either an input or an output port. It is useful in the wildcard
-specification described below.</p>
-<h3 id="stream-properties">Stream properties</h3>
-<p>Streams can be configured using stream properties. The properties can be
-specified using the parameter
-<code>dt.stream.<stream-name>.prop.<property-name></code> The constant \u201cstream\u201d
-specifies that it is a stream, <code><stream-name></code> specifies the name of the
-stream and <code><property-name></code> the name of the property. The name of the
-stream is the same name that is passed when the stream is added to the
-DAG using the addStream method. An example illustrating the
-specification is shown below. It sets the locality of the stream named
-\u201cstream1\u201d to container local indicating that the operators the stream is
-connecting be run in the same container.</p>
-<pre><code> <property>
- <name>dt.stream.stream1.prop.locality</name>
- <value>CONTAINER_LOCAL</value>
- </property>
-</code></pre>
-
-<p>The property name is converted into a set method on the stream in the
-same way as described in operator properties section above. In this case
-the method would be setLocality and it will be called in the stream
-\u201cstream1\u201d with the value as the argument.</p>
-<p>Along with the above system defined parameters, the applications can
-define their own specific parameters they can be specified in the
-configuration file. The only condition is that the names of these
-parameters don\u2019t conflict with the system defined parameters or similar
-application parameters defined by other applications. To this end, it is
-recommended that the application parameters have the format
-<code><full-application-class-name>.<param-name>.</code> The
-full-application-class-name is the full JAVA class name of the
-application including the package path and param-name is the name of the
-parameter within the application. The application will still have to
-still read the parameter in using the configuration API of the
-configuration object that is passed in populateDAG.</p>
-<h3 id="wildcards">Wildcards</h3>
-<p>Wildcards and regular expressions can be used in place of names to
-specify a group for applications, operators, ports or streams. For
-example, to specify an attribute for all ports of an operator it can be
-done as follows</p>
-<pre><code><property>
- <name>dt.operator.range.port.*.attr.QUEUE_CAPACITY</name>
- <value>4000</value>
-</property>
-</code></pre>
-
-<p>The wildcard \u201c*\u201d was used instead of the name of the port. Wildcard can
-also be used for operator name, stream name or application name. Regular
-expressions can also be used for names to specify attributes or
-properties for a specific set.</p>
-<h3 id="adding-configuration-properties">Adding configuration properties</h3>
-<p>It is common for applications to require configuration parameters to
-run. For example, the address and port of the database, the location of
-a file for ingestion, etc. You can specify them in
-src/main/resources/META-INF/properties.xml under the App Package
-project. The properties.xml may look like:</p>
-<pre><code><?xml version="1.0"?>
-<configuration>
- <property>
- <name>some_name_1</name>
- </property>
- <property>
- <name>some_name_2</name>
- <value>some_default_value</value>
- </property>
-</configuration>
-</code></pre>
-
-<p>The name of an application-specific property takes the form of:</p>
-<p><code>dt.operator.{opName}.prop.{propName}</code></p>
-<p>The first represents the property with name propName of operator opName.
- Or you can set the application name at run time by setting this
-property:</p>
-<pre><code> dt.attr.APPLICATION_NAME
-</code></pre>
-<p>In this example, property some_name_1 is a required property which
-must be set at launch time, or it must be set by a pre-set configuration
-(see next section). Property some_name_2 is a property that is
-assigned with value some_default_value unless it is overridden at
-launch time.</p>
-<h3 id="adding-pre-set-configurations">Adding pre-set configurations</h3>
-<p>At build time, you can add pre-set configurations to the App Package by
-adding configuration XML files under <code>src/site/conf/<conf>.xml</code>in your
-project. You can then specify which configuration to use at launch
-time. The configuration XML is of the same format of the properties.xml
-file.</p>
-<h3 id="application-specific-properties-file">Application-specific properties file</h3>
-<p>You can also specify properties.xml per application in the application
-package. Just create a file with the name properties-{appName}.xml and
-it will be picked up when you launch the application with the specified
-name within the application package. In short:</p>
-<p>properties.xml: Properties that are global to the Configuration
-Package</p>
-<p>properties-{appName}.xml: Properties that are specific when launching
-an application with the specified appName.</p>
-<h3 id="properties-source-precedence">Properties source precedence</h3>
-<p>If properties with the same key appear in multiple sources (e.g. from
-app package default configuration as META-INF/properties.xml, from app
-package configuration in the conf directory, from launch time defines,
-etc), the precedence of sources, from highest to lowest, is as follows:</p>
-<ol>
-<li>Launch time defines (using -D option in CLI)</li>
-<li>Launch time specified configuration file in file system (using -conf
- option in CLI)</li>
-<li>Launch time specified package configuration (using -apconf option in
- CLI)</li>
-<li>Configuration from \$HOME/.dt/dt-site.xml</li>
-<li>Application defaults within the package as
- META-INF/properties-{appname}.xml</li>
-<li>Package defaults as META-INF/properties.xml</li>
-<li>dt-site.xml in local DT installation</li>
-<li>dt-site.xml stored in HDFS</li>
-</ol>
-<h3 id="other-meta-data">Other meta-data</h3>
-<p>In a Apex App Package project, the pom.xml file contains a
-section that looks like:</p>
-<pre><code><properties>
- <apex.version>3.2.0-incubating</apex.version>
- <apex.apppackage.classpath\>lib*.jar</apex.apppackage.classpath>
-</properties>
-</code></pre>
-
-<p>apex.version is the Apache Apex version that are to be used
-with this Application Package.</p>
-<p>apex.apppackage.classpath is the classpath that is used when
-launching the application in the Application Package. The default is
-lib/*.jar, where lib is where all the dependency jars are kept within
-the Application Package. One reason to change this field is when your
-Application Package needs the classpath in a specific order.</p>
-<h3 id="logging-configuration">Logging configuration</h3>
-<p>Just like other Java projects, you can change the logging configuration
-by having your log4j.properties under src/main/resources. For example,
-if you have the following in src/main/resources/log4j.properties:</p>
-<pre><code> log4j.rootLogger=WARN,CONSOLE
- log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
- log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
- log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p
- %c{2} %M - %m%n
-</code></pre>
-
-<p>The root logger\u2019s level is set to WARN and the output is set to the console (stdout).</p>
-<p>Note that by default from project created from the maven archetype,
-there is already a log4j.properties file under src/test/resources and
-that file is only used for the unit test.</p>
-<h2 id="zip-structure-of-application-package">Zip Structure of Application Package</h2>
-<p>Apache Apex Application Package files are zip files. You can examine the content of any Application Package by using unzip -t on your Linux command line.</p>
-<p>There are four top level directories in an Application Package:</p>
-<ol>
-<li>"app" contains the jar files of the DAG code and any custom operators, and any JSON or properties files that specify a DAG.</li>
-<li>"lib" contains all dependency jars</li>
-<li>"conf" contains all the pre-set configuration XML files.</li>
-<li>"META-INF" contains the MANIFEST.MF file and the properties.xml file.</li>
-<li>\u201cresources\u201d contains any other files</li>
-</ol>
-<h2 id="examining-and-launching-application-packages-through-cli">Examining and Launching Application Packages Through CLI</h2>
-<p>If you are working with Application Packages in the local filesystem, you can use the Apex Command Line Interface (dtcli). </p>
-<h3 id="getting-application-package-meta-information">Getting Application Package Meta Information</h3>
-<p>You can get the meta information about the Application Package using
-this Apex CLI command.</p>
-<pre><code> dt> get-app-package-info <app-package-file>
-</code></pre>
-
-<h3 id="getting-available-operators-in-application-package">Getting Available Operators In Application Package</h3>
-<p>You can get the list of available operators in the Application Package
-using this command.</p>
-<pre><code> dt> get-app-package-operators <app-package-file> <package-prefix>
- [parent-class]
-</code></pre>
-
-<h3 id="getting-properties-of-operators-in-application-package">Getting Properties of Operators in Application Package</h3>
-<p>You can get the list of properties of any operator in the Application
-Package using this command.</p>
-<p>dt> get-app-package-operator-properties <app-package-file> <operator-class></p>
-<h3 id="launching-an-application-package">Launching an Application Package</h3>
-<p>You can launch an application within an Application Package.</p>
-<pre><code>dt> launch [-D property-name=property-value, ...] [-conf config-name]
- [-apconf config-file-within-app-package] <app-package-file>
- [matching-app-name]
-</code></pre>
-
-<p>Note that -conf expects a configuration file in the file system, while -apconf expects a configuration file within the app package.</p>
-<h1 id="configuration-packages">Configuration Packages</h1>
-<p>Sometimes just a configuration file is not enough for launching an application package. If a configuration requires
-additional files to be packaged, you can use an Apex Configuration Package.</p>
-<h2 id="creating-configuration-packages">Creating Configuration Packages</h2>
-<p>Creating Configuration Packages is similar to creating Application Packages. You can create a configuration
-package project using Maven by running the following command. Replace "com.example", "mydtconfig" and "1.0-SNAPSHOT" with the appropriate values:</p>
-<pre><code>$ mvn archetype:generate -DarchetypeGroupId=org.apache.apex \
- -DarchetypeArtifactId=apex-conf-archetype -DarchetypeVersion=3.2.0-incubating \
- -DgroupId=com.example -Dpackage=com.example.mydtconfig -DartifactId=mydtconfig \
- -Dversion=1.0-SNAPSHOT
-</code></pre>
-
-<p>And create the configuration package file by running:</p>
-<pre><code>$ mvn package
-</code></pre>
-
-<p>The "mvn package" command creates the Config Package file in target
-directory as target/mydtconfig.apc. You will be able to use that
-Configuration Package file to launch an Apache Apex application.</p>
-<h2 id="assembling-your-own-configuration-package">Assembling your own configuration package</h2>
-<p>Inside the project created by the archetype, these are the files that
-you should know about when assembling your own configuration package:</p>
-<pre><code>./pom.xml
-./src/main/resources/classpath
-./src/main/resources/files
-./src/main/resources/META-INF/properties.xml
-./src/main/resources/META-INF/properties-{appname}.xml
-</code></pre>
-<h3 id="pomxml">pom.xml</h3>
-<p>Example:</p>
-<pre><code class="xml"> <groupId>com.example</groupId>
- <version>1.0.0</version>
- <artifactId>mydtconf</artifactId>
- <packaging>jar</packaging>
- <!-- change these to the appropriate values -->
- <name>My Apex Application Configuration</name>
- <description>My Custom Application Configuration Description</description>
- <properties>
- <apex.apppackage.name>myapexapp</apex.apppackage.name>
- <apex.apppackage.minversion>1.0.0</apex.apppackage.minversion>
- <apex.apppackage.maxversion>1.9999.9999</apex.apppackage.maxversion>
- <apex.appconf.classpath>classpath/*</apex.appconf.classpath>
- <apex.appconf.files>files/*</apex.appconf.files>
- </properties>
-
-</code></pre>
-
-<p>In pom.xml, you can change the following keys to your desired values</p>
-<ul>
-<li><code><groupId></code></li>
-<li><code><version></code></li>
-<li><code><artifactId></code></li>
-<li><code><name></code></li>
-<li><code><description></code></li>
-</ul>
-<p>You can also change the values of</p>
-<ul>
-<li><code><apex.apppackage.name></code></li>
-<li><code><apex.apppackage.minversion></code></li>
-<li><code><apex.apppackage.maxversion></code></li>
-</ul>
-<p>to reflect what Application Packages can be used with this configuration package. Apex will use this information to check whether a
-configuration package is compatible with the Application Package when you issue a launch command.</p>
-<h3 id="srcmainresourcesclasspath">./src/main/resources/classpath</h3>
-<p>Place any file in this directory that you\u2019d like to be copied to the
-compute machines when launching an application and included in the
-classpath of the application. Example of such files are Java properties
-files and jar files.</p>
-<h3 id="srcmainresourcesfiles">./src/main/resources/files</h3>
-<p>Place any file in this directory that you\u2019d like to be copied to the
-compute machines when launching an application but not included in the
-classpath of the application.</p>
-<h3 id="properties-xml-file">Properties XML file</h3>
-<p>A properties xml file consists of a set of key-value pairs. The set of
-key-value pairs specifies the configuration options the application
-should be launched with.</p>
-<p>Example:</p>
-<pre><code class="xml"><configuration>
- <property>
- <name>some-property-name</name>
- <value>some-property-value</value>
- </property>
- ...
-</configuration>
-</code></pre>
-
-<p>Names of properties XML file:</p>
-<ul>
-<li><strong>properties.xml:</strong> Properties that are global to the Configuration
-Package</li>
-<li><strong>properties-{appName}.xml:</strong> Properties that are specific when launching
-an application with the specified appName within the Application
-Package.</li>
-</ul>
-<p>After you are done with the above, remember to do mvn package to
-generate a new configuration package, which will be located in the
-target directory in your project.</p>
-<h3 id="zip-structure-of-configuration-package">Zip structure of configuration package</h3>
-<p>Apex Application Configuration Package files are zip files. You
-can examine the content of any Application Configuration Package by
-using unzip -t on your Linux command line. The structure of the zip
-file is as follow:</p>
-<pre><code>META-INF
- MANIFEST.MF
- properties.xml
- properties-{appname}.xml
-classpath
- {classpath files}
-files
- {files}
-</code></pre>
-
-<h3 id="launching-with-cli">Launching with CLI</h3>
-<p><code>-conf</code> option of the launch command in CLI supports specifying configuration package in the local filesystem. Example:</p>
-<pre><code>dt\> launch mydtapp-1.0.0.apa -conf mydtconfig.apc
-</code></pre>
-<p>This command expects both the application package and the configuration package to be in the local file system.</p>
-
- </div>
- </div>
- <footer>
-
- <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
-
- <a href="../operator_development/" class="btn btn-neutral float-right" title="Operators">Next <span class="icon icon-circle-arrow-right"></span></a>
-
-
- <a href="../application_development/" class="btn btn-neutral" title="Applications"><span class="icon icon-circle-arrow-left"></span> Previous</a>
-
- </div>
-
-
- <hr/>
-
- <div role="contentinfo">
- <!-- Copyright etc -->
-
- </div>
-
- Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-<div class="rst-versions" role="note" style="cursor: pointer">
- <span class="rst-current-version" data-toggle="rst-current-version">
-
-
- <span><a href="../application_development/" style="color: #fcfcfc;">« Previous</a></span>
-
-
- <span style="margin-left: 15px"><a href="../operator_development/" style="color: #fcfcfc">Next »</a></span>
-
- </span>
-</div>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/1052900a/docs/apex-3.2/autometrics/index.html
----------------------------------------------------------------------
diff --git a/docs/apex-3.2/autometrics/index.html b/docs/apex-3.2/autometrics/index.html
deleted file mode 100644
index d52c369..0000000
--- a/docs/apex-3.2/autometrics/index.html
+++ /dev/null
@@ -1,398 +0,0 @@
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-
-
- <title>AutoMetric API - Apache Apex Documentation</title>
-
-
- <link rel="shortcut icon" href="../favicon.ico">
-
-
-
- <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
-
- <link rel="stylesheet" href="../css/theme.css" type="text/css" />
- <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
- <link rel="stylesheet" href="../css/highlight.css">
-
-
- <script>
- // Current page data
- var mkdocs_page_name = "AutoMetric API";
- var mkdocs_page_input_path = "autometrics.md";
- var mkdocs_page_url = "/autometrics/";
- </script>
-
- <script src="../js/jquery-2.1.1.min.js"></script>
- <script src="../js/modernizr-2.8.3.min.js"></script>
- <script type="text/javascript" src="../js/highlight.pack.js"></script>
- <script src="../js/theme.js"></script>
-
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
- <div class="wy-side-nav-search">
- <a href=".." class="icon icon-home"> Apache Apex Documentation</a>
- <div role="search">
- <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- </form>
-</div>
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
- <ul class="current">
-
- <li>
- <li class="toctree-l1 ">
- <a class="" href="..">Apache Apex</a>
-
- </li>
-<li>
-
- <li>
- <ul class="subnav">
- <li><span>Development</span></li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../apex_development_setup/">Development Setup</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../application_development/">Applications</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../application_packages/">Packages</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../operator_development/">Operators</a>
-
- </li>
-
-
-
- <li class="toctree-l1 current">
- <a class="current" href="./">AutoMetric API</a>
-
- <ul>
-
- <li class="toctree-l3"><a href="#apache-apex-autometrics">Apache Apex AutoMetrics</a></li>
-
-
- <li class="toctree-l3"><a href="#introduction">Introduction</a></li>
-
-
- <li class="toctree-l3"><a href="#specifying-autometrics-in-an-operator">Specifying AutoMetrics in an Operator</a></li>
-
-
- <li class="toctree-l3"><a href="#aggregating-autometrics-across-partitions">Aggregating AutoMetrics across Partitions</a></li>
-
- <li><a class="toctree-l4" href="#default-aggregators">Default aggregators</a></li>
-
- <li><a class="toctree-l4" href="#building-custom-aggregators">Building custom aggregators</a></li>
-
-
- <li class="toctree-l3"><a href="#retrieving-autometrics">Retrieving AutoMetrics</a></li>
-
-
- <li class="toctree-l3"><a href="#system-metrics">System Metrics</a></li>
-
-
- </ul>
-
- </li>
-
-
- </ul>
-<li>
-
- <li>
- <ul class="subnav">
- <li><span>Operations</span></li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../dtcli/">dtCli</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../security/">Security</a>
-
- </li>
-
-
- </ul>
-<li>
-
- <li>
- <li class="toctree-l1 ">
- <a class="" href="../compatibility/">Compatibility</a>
-
- </li>
-<li>
-
- </ul>
- </div>
-
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="..">Apache Apex Documentation</a>
- </nav>
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
- <div role="navigation" aria-label="breadcrumbs navigation">
- <ul class="wy-breadcrumbs">
- <li><a href="..">Docs</a> »</li>
-
-
-
- <li>Development »</li>
-
-
-
- <li>AutoMetric API</li>
- <li class="wy-breadcrumbs-aside">
-
- </li>
- </ul>
- <hr/>
-</div>
- <div role="main">
- <div class="section">
-
- <h1 id="apache-apex-autometrics">Apache Apex AutoMetrics</h1>
-<h1 id="introduction">Introduction</h1>
-<p>Metrics collect various statistical information about a process which can be very useful for diagnosis. Auto Metrics in Apex can help monitor operators in a running application. The goal of <em>AutoMetric</em> API is to enable operator developer to define relevant metrics for an operator in a simple way which the platform collects and reports automatically.</p>
-<h1 id="specifying-autometrics-in-an-operator">Specifying AutoMetrics in an Operator</h1>
-<p>An <em>AutoMetric</em> can be any object. It can be of a primitive type - int, long, etc. or a complex one. A field or a <code>get</code> method in an operator can be annotated with <code>@AutoMetric</code> to specify that its value is a metric. After every application end window, the platform collects the values of these fields/methods in a map and sends it to application master.</p>
-<pre><code class="java">public class LineReceiver extends BaseOperator
-{
- @AutoMetric
- long length;
-
- @AutoMetric
- long count;
-
- public final transient DefaultInputPort<String> input = new DefaultInputPort<String>()
- {
- @Override
- public void process(String s)
- {
- length += s.length();
- count++;
- }
- };
-
- @Override
- public void beginWindow(long windowId)
- {
- length = 0;
- count = 0;
- }
-}
-</code></pre>
-
-<p>There are 2 auto-metrics declared in the <code>LineReceiver</code>. At the end of each application window, the platform will send a map with 2 entries - <code>[(length, 100), (count, 10)]</code> to the application master.</p>
-<h1 id="aggregating-autometrics-across-partitions">Aggregating AutoMetrics across Partitions</h1>
-<p>When an operator is partitioned, it is useful to aggregate the values of auto-metrics across all its partitions every window to get a logical view of these metrics. The application master performs these aggregations using metrics aggregators.</p>
-<p>The AutoMetric API helps to achieve this by providing an interface for writing aggregators- <code>AutoMetric.Aggregator</code>. Any implementation of <code>AutoMetric.Aggregator</code> can be set as an operator attribute - <code>METRICS_AGGREGATOR</code> for a particular operator which in turn is used for aggregating physical metrics.</p>
-<h2 id="default-aggregators">Default aggregators</h2>
-<p><a href="https://github.com/apache/incubator-apex-core/blob/devel-3/common/src/main/java/com/datatorrent/common/metric/MetricsAggregator.java"><code>MetricsAggregator</code></a> is a simple implementation of <code>AutoMetric.Aggregator</code> that platform uses as a default for summing up primitive types - int, long, float and double.</p>
-<p><code>MetricsAggregator</code> is just a collection of <code>SingleMetricAggregator</code>s. There are multiple implementations of <code>SingleMetricAggregator</code> that perform sum, min, max, avg which are present in Apex core and Apex malhar.</p>
-<p>For the <code>LineReceiver</code> operator, the application developer need not specify any aggregator. The platform will automatically inject an instance of <code>MetricsAggregator</code> that contains two <code>LongSumAggregator</code>s - one for <code>length</code> and one for <code>count</code>. This aggregator will report sum of length and sum of count across all the partitions of <code>LineReceiver</code>.</p>
-<h2 id="building-custom-aggregators">Building custom aggregators</h2>
-<p>Platform cannot perform any meaningful aggregations for non-numeric metrics. In such cases, the operator or application developer can write custom aggregators. Let\u2019s say, if the <code>LineReceiver</code> was modified to have a complex metric as shown below.</p>
-<pre><code class="java">public class AnotherLineReceiver extends BaseOperator
-{
- @AutoMetric
- final LineMetrics lineMetrics = new LineMetrics();
-
- public final transient DefaultInputPort<String> input = new DefaultInputPort<String>()
- {
- @Override
- public void process(String s)
- {
- lineMetrics.length += s.length();
- lineMetrics.count++;
- }
- };
-
- @Override
- public void beginWindow(long windowId)
- {
- lineMetrics.length = 0;
- lineMetrics.count = 0;
- }
-
- public static class LineMetrics implements Serializable
- {
- long length;
- long count;
-
- private static final long serialVersionUID = 201511041908L;
- }
-}
-</code></pre>
-
-<p>Below is a custom aggregator that can calculate average line length across all partitions of <code>AnotherLineReceiver</code>.</p>
-<pre><code class="java">public class AvgLineLengthAggregator implements AutoMetric.Aggregator
-{
-
- Map<String, Object> result = Maps.newHashMap();
-
- @Override
- public Map<String, Object> aggregate(long l, Collection<AutoMetric.PhysicalMetricsContext> collection)
- {
- long totalLength = 0;
- long totalCount = 0;
- for (AutoMetric.PhysicalMetricsContext pmc : collection) {
- AnotherLineReceiver.LineMetrics lm = (AnotherLineReceiver.LineMetrics)pmc.getMetrics().get("lineMetrics");
- totalLength += lm.length;
- totalCount += lm.count;
- }
- result.put("avgLineLength", totalLength/totalCount);
- return result;
- }
-}
-</code></pre>
-
-<p>An instance of above aggregator can be specified as the <code>METRIC_AGGREGATOR</code> for <code>AnotherLineReceiver</code> while creating the DAG as shown below.</p>
-<pre><code class="java"> @Override
- public void populateDAG(DAG dag, Configuration configuration)
- {
- ...
- AnotherLineReceiver lineReceiver = dag.addOperator("LineReceiver", new AnotherLineReceiver());
- dag.setAttribute(lineReceiver, Context.OperatorContext.METRICS_AGGREGATOR, new AvgLineLengthAggregator());
- ...
- }
-</code></pre>
-
-<h1 id="retrieving-autometrics">Retrieving AutoMetrics</h1>
-<p>There are two options for retrieving the AutoMetrics:</p>
-<ul>
-<li>Throught DataTorrent Gateway REST API</li>
-<li>Through REST service on the port of the running STRAM</li>
-</ul>
-<p>The Gateway REST API provides a way to retrieve the latest AutoMetrics for each logical operator. For example:</p>
-<pre><code>GET /ws/v2/applications/{appid}/logicalPlan/operators/{opName}
-{
- ...
- "autoMetrics": {
- "count": "71314",
- "length": "27780706"
- },
- "className": "com.datatorrent.autometric.LineReceiver",
- ...
-}
-</code></pre>
-
-<h1 id="system-metrics">System Metrics</h1>
-<p>System metrics are standard operator metrics provided by the system. Examples include:</p>
-<ul>
-<li>processed tuples per second</li>
-<li>emitted tuples per second</li>
-<li>total tuples processed</li>
-<li>total tuples emitted</li>
-<li>latency</li>
-<li>CPU percentage</li>
-<li>failure count</li>
-<li>checkpoint elapsed time</li>
-</ul>
-<p>The Gateway REST API provides a way to retrieve the latest values for all of the above for each of the logical operators in the application.</p>
-<pre><code>GET /ws/v2/applications/{appid}/logicalPlan/operators/{opName}
-{
- ...
- "cpuPercentageMA": "{cpuPercentageMA}",
- "failureCount": "{failureCount}",
- "latencyMA": "{latencyMA}",
- "totalTuplesEmitted": "{totalTuplesEmitted}",
- "totalTuplesProcessed": "{totalTuplesProcessed}",
- "tuplesEmittedPSMA": "{tuplesEmittedPSMA}",
- "tuplesProcessedPSMA": "{tuplesProcessedPSMA}",
- ...
-}
-</code></pre>
-
-<p>However, just like AutoMetrics, the Gateway only provides the latest metrics. For historical metrics, we will need the help of <a href="http://docs.datatorrent.com/autometrics/#app-data-tracker">App Data Tracker</a>.</p>
-
- </div>
- </div>
- <footer>
-
- <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
-
- <a href="../dtcli/" class="btn btn-neutral float-right" title="dtCli">Next <span class="icon icon-circle-arrow-right"></span></a>
-
-
- <a href="../operator_development/" class="btn btn-neutral" title="Operators"><span class="icon icon-circle-arrow-left"></span> Previous</a>
-
- </div>
-
-
- <hr/>
-
- <div role="contentinfo">
- <!-- Copyright etc -->
-
- </div>
-
- Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-<div class="rst-versions" role="note" style="cursor: pointer">
- <span class="rst-current-version" data-toggle="rst-current-version">
-
-
- <span><a href="../operator_development/" style="color: #fcfcfc;">« Previous</a></span>
-
-
- <span style="margin-left: 15px"><a href="../dtcli/" style="color: #fcfcfc">Next »</a></span>
-
- </span>
-</div>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/1052900a/docs/apex-3.2/base.html
----------------------------------------------------------------------
diff --git a/docs/apex-3.2/base.html b/docs/apex-3.2/base.html
deleted file mode 100644
index 87c9f8a..0000000
--- a/docs/apex-3.2/base.html
+++ /dev/null
@@ -1,118 +0,0 @@
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
- {% if page_description %}<meta name="description" content="{{ page_description }}">{% endif %}
- {% if site_author %}<meta name="author" content="{{ site_author }}">{% endif %}
- {% block htmltitle %}
- <title>{% if page_title %}{{ page_title }} - {% endif %}{{ site_name }}</title>
- {% endblock %}
-
- {% if favicon %}<link rel="shortcut icon" href="{{ favicon }}">
- {% else %}<link rel="shortcut icon" href="{{ base_url }}/img/favicon.ico">{% endif %}
-
- {# CSS #}
- <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
-
- <link rel="stylesheet" href="{{ base_url }}/css/theme.css" type="text/css" />
- <link rel="stylesheet" href="{{ base_url }}/css/theme_extra.css" type="text/css" />
- <link rel="stylesheet" href="{{ base_url }}/css/highlight.css">
- {%- for path in extra_css %}
- <link href="{{ path }}" rel="stylesheet">
- {%- endfor %}
-
- {% if current_page %}
- <script>
- // Current page data
- var mkdocs_page_name = {{ page_title|tojson|safe }};
- var mkdocs_page_input_path = {{ current_page.input_path|tojson|safe }};
- var mkdocs_page_url = {{ current_page.abs_url|tojson|safe }};
- </script>
- {% endif %}
- <script src="{{ base_url }}/js/jquery-2.1.1.min.js"></script>
- <script src="{{ base_url }}/js/modernizr-2.8.3.min.js"></script>
- <script type="text/javascript" src="{{ base_url }}/js/highlight.pack.js"></script>
- <script src="{{ base_url }}/js/theme.js"></script>
-
- {%- block extrahead %} {% endblock %}
-
- {%- for path in extra_javascript %}
- <script src="{{ path }}"></script>
- {%- endfor %}
-
- {% if google_analytics %}
- <script>
- (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
- (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
- m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
- })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
-
- ga('create', '{{ google_analytics[0] }}', '{{ google_analytics[1] }}');
- ga('send', 'pageview');
- </script>
- {% endif %}
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
- <div class="wy-grid-for-nav">
-
- {# SIDE NAV, TOGGLES ON MOBILE #}
- <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
- <div class="wy-side-nav-search">
- <a href="{{ homepage_url }}" class="icon icon-home"> {{ site_name }}</a>
- {% include "searchbox.html" %}
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
- <ul class="current">
- {% for nav_item in nav %}
- <li>{% include "toc.html" %}<li>
- {% endfor %}
- </ul>
- </div>
-
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
- {# MOBILE NAV, TRIGGLES SIDE NAV ON TOGGLE #}
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="{{ homepage_url }}">{{ site_name }}</a>
- </nav>
-
- {# PAGE CONTENT #}
- <div class="wy-nav-content">
- <div class="rst-content">
- {% include "breadcrumbs.html" %}
- <div role="main">
- <div class="section">
- {% block content %}
- {{ content }}
- {% endblock %}
- </div>
- </div>
- {%- block footer %}
- {% include "footer.html" %}
- {% endblock %}
- </div>
- </div>
-
- </section>
-
- </div>
-
-{% include "versions.html" %}
-
-</body>
-</html>
-{% if current_page and current_page.is_homepage %}
-<!--
-MkDocs version : {{ mkdocs_version }}
-Build Date UTC : {{ build_date_utc }}
--->
-{% endif %}
http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/1052900a/docs/apex-3.2/breadcrumbs.html
----------------------------------------------------------------------
diff --git a/docs/apex-3.2/breadcrumbs.html b/docs/apex-3.2/breadcrumbs.html
deleted file mode 100644
index 01960e6..0000000
--- a/docs/apex-3.2/breadcrumbs.html
+++ /dev/null
@@ -1,25 +0,0 @@
-<div role="navigation" aria-label="breadcrumbs navigation">
- <ul class="wy-breadcrumbs">
- <li><a href="{{ homepage_url }}">Docs</a> »</li>
- {% if current_page %}
- {% for doc in current_page.ancestors %}
- {% if doc.link %}
- <li><a href="{{ doc.link|e }}">{{ doc.title }}</a> »</li>
- {% else %}
- <li>{{ doc.title }} »</li>
- {% endif %}
- {% endfor %}
- {% endif %}
- {% if current_page %}<li>{{ current_page.title }}</li>{% endif %}
- <li class="wy-breadcrumbs-aside">
- {% if repo_url %}
- {% if repo_name == 'GitHub' %}
- <a href="{{ repo_url }}" class="icon icon-github"> Edit on GitHub</a>
- {% elif repo_name == 'Bitbucket' %}
- <a href="{{ repo_url }}" class="icon icon-bitbucket"> Edit on BitBucket</a>
- {% endif %}
- {% endif %}
- </li>
- </ul>
- <hr/>
-</div>
http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/1052900a/docs/apex-3.2/compatibility/index.html
----------------------------------------------------------------------
diff --git a/docs/apex-3.2/compatibility/index.html b/docs/apex-3.2/compatibility/index.html
deleted file mode 100644
index 42ded9a..0000000
--- a/docs/apex-3.2/compatibility/index.html
+++ /dev/null
@@ -1,287 +0,0 @@
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-
-
- <title>Compatibility - Apache Apex Documentation</title>
-
-
- <link rel="shortcut icon" href="../favicon.ico">
-
-
-
- <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
-
- <link rel="stylesheet" href="../css/theme.css" type="text/css" />
- <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
- <link rel="stylesheet" href="../css/highlight.css">
-
-
- <script>
- // Current page data
- var mkdocs_page_name = "Compatibility";
- var mkdocs_page_input_path = "compatibility.md";
- var mkdocs_page_url = "/compatibility/";
- </script>
-
- <script src="../js/jquery-2.1.1.min.js"></script>
- <script src="../js/modernizr-2.8.3.min.js"></script>
- <script type="text/javascript" src="../js/highlight.pack.js"></script>
- <script src="../js/theme.js"></script>
-
-
-</head>
-
-<body class="wy-body-for-nav" role="document">
-
- <div class="wy-grid-for-nav">
-
-
- <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
- <div class="wy-side-nav-search">
- <a href=".." class="icon icon-home"> Apache Apex Documentation</a>
- <div role="search">
- <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
- <input type="text" name="q" placeholder="Search docs" />
- </form>
-</div>
- </div>
-
- <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
- <ul class="current">
-
- <li>
- <li class="toctree-l1 ">
- <a class="" href="..">Apache Apex</a>
-
- </li>
-<li>
-
- <li>
- <ul class="subnav">
- <li><span>Development</span></li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../apex_development_setup/">Development Setup</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../application_development/">Applications</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../application_packages/">Packages</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../operator_development/">Operators</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../autometrics/">AutoMetric API</a>
-
- </li>
-
-
- </ul>
-<li>
-
- <li>
- <ul class="subnav">
- <li><span>Operations</span></li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../dtcli/">dtCli</a>
-
- </li>
-
-
-
- <li class="toctree-l1 ">
- <a class="" href="../security/">Security</a>
-
- </li>
-
-
- </ul>
-<li>
-
- <li>
- <li class="toctree-l1 current">
- <a class="current" href="./">Compatibility</a>
-
- <ul>
-
- <li class="toctree-l3"><a href="#apache-apex-compatibility">Apache Apex Compatibility</a></li>
-
- <li><a class="toctree-l4" href="#purpose">Purpose</a></li>
-
- <li><a class="toctree-l4" href="#compatibility-types">Compatibility types</a></li>
-
-
- </ul>
-
- </li>
-<li>
-
- </ul>
- </div>
-
- </nav>
-
- <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="..">Apache Apex Documentation</a>
- </nav>
-
-
- <div class="wy-nav-content">
- <div class="rst-content">
- <div role="navigation" aria-label="breadcrumbs navigation">
- <ul class="wy-breadcrumbs">
- <li><a href="..">Docs</a> »</li>
-
-
-
- <li>Compatibility</li>
- <li class="wy-breadcrumbs-aside">
-
- </li>
- </ul>
- <hr/>
-</div>
- <div role="main">
- <div class="section">
-
- <h1 id="apache-apex-compatibility">Apache Apex Compatibility</h1>
-<h2 id="purpose">Purpose</h2>
-<p>This document captures the compatibility goals of the Apache Apex project. The different types of compatibility between Apex releases that affect contributors, downstream projects, and end-users are enumerated. For each type of compatibility we:</p>
-<ul>
-<li>describe the impact on downstream projects or end-users</li>
-<li>where applicable, call out the policy adopted when incompatible changes are permitted.</li>
-</ul>
-<p>Apache Apex follows <a href="http://semver.org/">semantic versioning</a>. Depending on the compatibility type, there may be different tools or mechanisms to ensure compatibility, for example by comparing artifacts during the build process.</p>
-<p>The type of change will inform the required target version number. Given a version number MAJOR.MINOR.PATCH, increment the:</p>
-<ul>
-<li>MAJOR version when you make incompatible API changes,</li>
-<li>MINOR version when you add functionality in a backward-compatible manner, and</li>
-<li>PATCH version when you make backward-compatible bug fixes.</li>
-</ul>
-<p>Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.</p>
-<p>The overall goal is to avoid backward incompatible changes and major release upgrades. Accordingly we attempt to release new features with minor versions that are incremental to the prior release and offer our users a frictionless upgrade path. When planning contributions, please consider compatibility and release road map upfront. Specifically, certain changes that conflict with the versioning may need to be documented in JIRA and deferred until a future major release. </p>
-<h2 id="compatibility-types">Compatibility types</h2>
-<h3 id="java-api">Java API</h3>
-<p>Public API compatibility is required to ensure end-user programs and downstream projects continue to work without modification.
-The public API consists of:</p>
-<ul>
-<li>apex-core: all interfaces and classes in <code>api</code> and <code>common</code> modules</li>
-<li>apex-malhar: all interfaces and classes in all modules except <code>demos</code>, <code>samples</code>, <code>benchmark</code> </li>
-</ul>
-<p>Interfaces and classes that are part of the public API and are annotated with <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/InterfaceClassification.html">interface stability</a> are treated according to the rules defined by the annotation. </p>
-<p>Policy</p>
-<p>Changes to the public API must follow semantic versioning.
-Public APIs must be deprecated for at least one minor release prior to their removal in a major release.
-The <a href="https://github.com/siom79/japicmp">japicmp Maven plugin</a> is used to enforce compatibility as part of the Travis pre-commit builds.</p>
-<h3 id="semantic-compatibility">Semantic compatibility</h3>
-<p>The behavior of APIs needs to remain consistent over versions, though changes for correctness may result in changes in behavior. Tests and javadocs specify the behavior. Over time, test suites should be expanded to verify compliance with the specification, effectively creating a formal specification for the subset of behaviors that can be easily tested.</p>
-<p>Policy</p>
-<p>The behavior of existing API cannot be modified as it would break existing user code. There are exceptional circumstances that may justify such changes, in which cases they should be discussed on the mailing list before implementation. Examples are bug fixes related to security issues, data corruption/consistency or to correct an unintended change from previous release that violated semantic compatibility. Such changes should be accompanied by test coverage for the exact behavior.</p>
-<h3 id="rest-api">REST API</h3>
-<p>REST API compatibility corresponds to both the URLs and request/response content over the wire. REST APIs are specifically meant for stable use by clients across releases, even major releases. </p>
-<p>Policy</p>
-<p>The REST API is separately versioned. This is to allow for co-existence of old and new API should there be a need for backward incompatible changes in the future.</p>
-<h3 id="command-line-interface-cli">Command Line Interface (CLI)</h3>
-<p>The CLI may be used either directly via the system shell or via shell scripts. Changing the path, removing or renaming command line options, the order of arguments, or the command return code and output break compatibility and may adversely affect users.</p>
-<p>Policy</p>
-<p>CLI commands are to be deprecated (warning when used) in a prior minor release before they are removed or incompatibly modified in a subsequent major release.</p>
-<h3 id="configuration-files">Configuration Files</h3>
-<p>Configuration files are used for engine or application settings. Changes to keys and default values directly affect users and are hard to diagnose (compared to a compile error, for example).</p>
-<p>Policy</p>
-<p>Name, location, format, keys of configuration files should be deprecated in a prior minor release and can only be changed in major release. Best effort should be made to support the deprecated behavior for one more major release (not guaranteed). It is also desirable to provide the user with a migration tool.</p>
-<h3 id="internal-wire-compatibility">Internal Wire compatibility</h3>
-<p>Apex containers internally use RPC communication and netlet for the data flow. The protocols are private and user components are not exposed to it. Apex is a YARN application and automatically deployed. There is currently no situation where containers of different Apex engine versions need to be interoperable. Should such a scenario become relevant in the future, wire compatibility needs to be specified.</p>
-<p>Policy</p>
-<p>N/A</p>
-<h3 id="internal-file-formats">Internal File formats</h3>
-<p>Apex engine stores data in the file system for recovery and the data is typically obtained from serialization (from Kryo, Java etc.). Changes to internal classes may affect the ability to relaunch an application with upgraded engine code from previous state. This is currently not supported. In the future, the serialization mechanism should guarantee backward compatibility.</p>
-<p>Policy</p>
-<p>Currently no compatibility guarantee. User to cold-restart application on engine upgrade.</p>
-<h3 id="java-classpath">Java Classpath</h3>
-<p>Apex applications should not bundle Hadoop dependencies or Apex engine dependencies but use the dependencies provided in the target environment to avoid conflicts. The Apex application archetype can be used to generate a compliant project. </p>
-<p>Policy</p>
-<p>Apex engine dependencies can change as per semantic versioning. Following above guidelines automatically maintains the backward compatibility based on semantic versioning of Apex.</p>
-<h3 id="maven-build-artifacts">Maven Build Artifacts</h3>
-<p>Downstream projects reference the Apex engine dependencies and Malhar operator libraries for application development etc. Changes to the packaging (which classes are in which jar), the groupId, artifactId and which artifacts are deployed to Maven central impact upgrades.</p>
-<p>Policy</p>
-<p>The artifacts that contain the classes that form the public API as specified above cannot change in patch releases and should stay compatible within a major release. Patch releases can change dependencies, but only at the patch level and following semantic versioning.</p>
-<h3 id="hardwaresoftware-requirements">Hardware/Software Requirements</h3>
-<p>Apex depends on Apache Hadoop. The community intends to support all major Hadoop distros and current versions. Apex currently supports Hadoop 2.2.0 and higher and Java 7 and higher. Apex is written in Java and has been tested on Linux based Hadoop clusters. There are no additional restrictions on the hardware architecture. </p>
-<p>To keep up with the latest advances in hardware, operating systems, JVMs, Hadoop and other software, new Apex releases may require higher versions. Upgrading Apex may require upgrading other dependent software components.</p>
-<p>Policy</p>
-<p>The JVM and Hadoop minimum version requirements are not expected to change outside major releases.</p>
-
- </div>
- </div>
- <footer>
-
- <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
-
-
- <a href="../security/" class="btn btn-neutral" title="Security"><span class="icon icon-circle-arrow-left"></span> Previous</a>
-
- </div>
-
-
- <hr/>
-
- <div role="contentinfo">
- <!-- Copyright etc -->
-
- </div>
-
- Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
-</footer>
-
- </div>
- </div>
-
- </section>
-
- </div>
-
-<div class="rst-versions" role="note" style="cursor: pointer">
- <span class="rst-current-version" data-toggle="rst-current-version">
-
-
- <span><a href="../security/" style="color: #fcfcfc;">« Previous</a></span>
-
-
- </span>
-</div>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/1052900a/docs/apex-3.2/css/highlight.css
----------------------------------------------------------------------
diff --git a/docs/apex-3.2/css/highlight.css b/docs/apex-3.2/css/highlight.css
deleted file mode 100644
index 0ae40a7..0000000
--- a/docs/apex-3.2/css/highlight.css
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
-This is the GitHub theme for highlight.js
-
-github.com style (c) Vasily Polovnyov <va...@whiteants.net>
-
-*/
-
-.hljs {
- display: block;
- overflow-x: auto;
- color: #333;
- -webkit-text-size-adjust: none;
-}
-
-.hljs-comment,
-.diff .hljs-header,
-.hljs-javadoc {
- color: #998;
- font-style: italic;
-}
-
-.hljs-keyword,
-.css .rule .hljs-keyword,
-.hljs-winutils,
-.nginx .hljs-title,
-.hljs-subst,
-.hljs-request,
-.hljs-status {
- color: #333;
- font-weight: bold;
-}
-
-.hljs-number,
-.hljs-hexcolor,
-.ruby .hljs-constant {
- color: #008080;
-}
-
-.hljs-string,
-.hljs-tag .hljs-value,
-.hljs-phpdoc,
-.hljs-dartdoc,
-.tex .hljs-formula {
- color: #d14;
-}
-
-.hljs-title,
-.hljs-id,
-.scss .hljs-preprocessor {
- color: #900;
- font-weight: bold;
-}
-
-.hljs-list .hljs-keyword,
-.hljs-subst {
- font-weight: normal;
-}
-
-.hljs-class .hljs-title,
-.hljs-type,
-.vhdl .hljs-literal,
-.tex .hljs-command {
- color: #458;
- font-weight: bold;
-}
-
-.hljs-tag,
-.hljs-tag .hljs-title,
-.hljs-rule .hljs-property,
-.django .hljs-tag .hljs-keyword {
- color: #000080;
- font-weight: normal;
-}
-
-.hljs-attribute,
-.hljs-variable,
-.lisp .hljs-body,
-.hljs-name {
- color: #008080;
-}
-
-.hljs-regexp {
- color: #009926;
-}
-
-.hljs-symbol,
-.ruby .hljs-symbol .hljs-string,
-.lisp .hljs-keyword,
-.clojure .hljs-keyword,
-.scheme .hljs-keyword,
-.tex .hljs-special,
-.hljs-prompt {
- color: #990073;
-}
-
-.hljs-built_in {
- color: #0086b3;
-}
-
-.hljs-preprocessor,
-.hljs-pragma,
-.hljs-pi,
-.hljs-doctype,
-.hljs-shebang,
-.hljs-cdata {
- color: #999;
- font-weight: bold;
-}
-
-.hljs-deletion {
- background: #fdd;
-}
-
-.hljs-addition {
- background: #dfd;
-}
-
-.diff .hljs-change {
- background: #0086b3;
-}
-
-.hljs-chunk {
- color: #aaa;
-}