You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by jg...@apache.org on 2014/07/09 22:06:55 UTC
git commit: SAMZA-285: Use Apache Rat for release audit tests
Repository: incubator-samza
Updated Branches:
refs/heads/master dd13c3d20 -> 9652e6412
SAMZA-285: Use Apache Rat for release audit tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-samza/commit/9652e641
Tree: http://git-wip-us.apache.org/repos/asf/incubator-samza/tree/9652e641
Diff: http://git-wip-us.apache.org/repos/asf/incubator-samza/diff/9652e641
Branch: refs/heads/master
Commit: 9652e6412f668bcd738b02caf77fe685fd56b46f
Parents: dd13c3d
Author: David Chen <dc...@linkedin.com>
Authored: Wed Jul 9 13:06:56 2014 -0700
Committer: Jakob Homan <jg...@apache.org>
Committed: Wed Jul 9 13:06:56 2014 -0700
----------------------------------------------------------------------
build.gradle | 53 ++++-
.../documentation/0.7.0/jobs/reprocessing.md | 16 ++
gradle/buildscript.gradle | 6 -
gradle/license.gradle | 32 ---
gradle/rat.gradle | 109 ++++++++++
gradle/resources/rat-output-to-html.xsl | 206 +++++++++++++++++++
6 files changed, 377 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index f1a458b..c853fc0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
buildscript {
repositories {
mavenCentral()
@@ -35,11 +36,50 @@ allprojects {
}
}
-apply from: file('gradle/license.gradle')
apply from: file('gradle/environment.gradle')
apply from: file("gradle/dependency-versions.gradle")
apply from: file("gradle/dependency-versions-scala-" + scalaVersion + ".gradle")
apply from: file("gradle/release.gradle")
+apply from: file("gradle/rat.gradle")
+
+rat {
+ excludes = [
+ '**/.git/**',
+ '**/.gradle/**',
+ '**/.project',
+ '**/.settings',
+ '**/*.iml',
+ '**/*.iws',
+ '**/*.ipr',
+ 'gradle/wrapper/**',
+ '.reviewboardrc',
+ 'gradlew',
+ 'README.md',
+ 'RELEASE.md',
+ '**/.DS_Store/**',
+ '**/.gitignore',
+ '**/build/**',
+ 'docs/Gemfile.lock',
+ 'docs/sitemap.xml',
+ 'docs/_site/**',
+ 'samza-test/state/mystore/**',
+ '**/bootstrap.min.js',
+ '**/jquery-1.11.1.min.js',
+ '**/jquery.tablesorter.min.js',
+ '**/bootstrap.css.map',
+ '**/bootstrap.min.css',
+ '**/ropa-sans.css',
+ '**/font-awesome.min.css',
+ '**/syntax.css',
+ '**/non-responsive.less',
+ '**/*.graffle',
+ '**/*.otf',
+ '**/*.woff',
+ '**/*.eot',
+ '**/*.svg',
+ '**/*.ttf'
+ ]
+}
allprojects {
apply plugin: 'idea'
@@ -200,8 +240,8 @@ project(":samza-shell") {
gradleShell "log4j:log4j:1.2.16"
}
- task shellTarGz(type: Tar) {
- compression = Compression.GZIP
+ task shellTarGz(type: Tar) {
+ compression = Compression.GZIP
classifier = 'dist'
from 'src/main/bash'
from 'src/main/resources'
@@ -284,16 +324,15 @@ project(":samza-test_$scalaVersion") {
}
test {
- // Bump up the heap so we can start ZooKeeper and Kafka brokers. Also
- // required for TestSamzaContainerPerformance when a high thread count
+ // Bump up the heap so we can start ZooKeeper and Kafka brokers. Also
+ // required for TestSamzaContainerPerformance when a high thread count
// with a lot of inputs is used.
minHeapSize = "4096m"
maxHeapSize = "4096m"
jvmArgs = ["-XX:+UseConcMarkSweepGC", "-server", "-d64"]
- // Forward all samza.* system properties to test subprocesses. This is
+ // Forward all samza.* system properties to test subprocesses. This is
// useful for configuring TestSamzaContainerPerformance from the CLI.
systemProperties = System.properties.findAll { it.key.startsWith("samza") }
}
}
-
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/docs/learn/documentation/0.7.0/jobs/reprocessing.md
----------------------------------------------------------------------
diff --git a/docs/learn/documentation/0.7.0/jobs/reprocessing.md b/docs/learn/documentation/0.7.0/jobs/reprocessing.md
index 4c92ef9..28d9925 100644
--- a/docs/learn/documentation/0.7.0/jobs/reprocessing.md
+++ b/docs/learn/documentation/0.7.0/jobs/reprocessing.md
@@ -2,6 +2,22 @@
layout: page
title: Reprocessing previously processed data
---
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
From time to time you may want to deploy a new version of your Samza job that computes results differently. Perhaps you fixed a bug or introduced a new feature. For example, say you have a Samza job that classifies messages as spam or not-spam, using a machine learning model that you train offline. Periodically you want to deploy an updated version of your Samza job which includes the latest classification model.
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/gradle/buildscript.gradle
----------------------------------------------------------------------
diff --git a/gradle/buildscript.gradle b/gradle/buildscript.gradle
index 739a9c1..a2bdb35 100644
--- a/gradle/buildscript.gradle
+++ b/gradle/buildscript.gradle
@@ -18,11 +18,6 @@
*/
repositories {
repositories {
- // For license plugin.
- maven {
- url 'http://dl.bintray.com/content/netflixoss/external-gradle-plugins/'
- }
-
// For gradle-nexus-plugin
maven {
url 'http://jcenter.bintray.com/'
@@ -31,7 +26,6 @@ repositories {
}
dependencies {
- classpath 'nl.javadude.gradle.plugins:license-gradle-plugin:0.6.1'
classpath 'de.obqo.gradle:gradle-lesscss-plugin:1.0-1.3.3'
classpath 'org.gradle.api.plugins:gradle-nexus-plugin:0.7.1'
}
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/gradle/license.gradle
----------------------------------------------------------------------
diff --git a/gradle/license.gradle b/gradle/license.gradle
deleted file mode 100644
index 82aca21..0000000
--- a/gradle/license.gradle
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-subprojects {
- apply plugin: 'license'
-
- license {
- header rootProject.file('HEADER')
- // Skip Twitter bootstrap JS and CSS.
- skipExistingHeaders = true
- mapping {
- // Treat integration scripts (with .samsa extension) as
- // regular scripts, as they use the same comment style
- samsa='SCRIPT_STYLE'
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/gradle/rat.gradle
----------------------------------------------------------------------
diff --git a/gradle/rat.gradle b/gradle/rat.gradle
new file mode 100644
index 0000000..8c8ac5e
--- /dev/null
+++ b/gradle/rat.gradle
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.internal.project.IsolatedAntBuilder
+
+apply plugin: RatPlugin
+
+class RatTask extends DefaultTask {
+ @Input
+ List<String> excludes
+
+ def reportPath = 'build/rat'
+ def stylesheet = 'gradle/resources/rat-output-to-html.xsl'
+ def xmlReport = reportPath + '/rat-report.xml'
+ def htmlReport = reportPath + '/rat-report.html'
+
+ def generateXmlReport(File reportDir) {
+ def antBuilder = services.get(IsolatedAntBuilder)
+ def ratClasspath = project.configurations.rat
+ antBuilder.withClasspath(ratClasspath).execute {
+ ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml')
+ ant.report(format: 'xml', reportFile: xmlReport) {
+ fileset(dir: ".") {
+ patternset {
+ excludes.each {
+ exclude(name: it)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ def printUnknownFiles() {
+ def ratXml = new XmlParser().parse(xmlReport)
+ ratXml.resource.each { resource ->
+ if (resource.'license-approval'.@name[0] == "false") {
+ println('Unknown license: ' + resource.@name)
+ }
+ }
+ }
+
+ def generateHtmlReport() {
+ def antBuilder = services.get(IsolatedAntBuilder)
+ def ratClasspath = project.configurations.rat
+ antBuilder.withClasspath(ratClasspath).execute {
+ ant.xslt(
+ in: xmlReport,
+ style: stylesheet,
+ out: htmlReport,
+ classpath: ratClasspath)
+ }
+ println('Rat report: ' + htmlReport)
+ }
+
+ @TaskAction
+ def rat() {
+ File reportDir = new File(reportPath)
+ if (!reportDir.exists()) {
+ reportDir.mkdirs()
+ }
+ generateXmlReport(reportDir)
+ printUnknownFiles()
+ generateHtmlReport()
+ }
+}
+
+class RatPlugin implements Plugin<Project> {
+ void apply(Project project) {
+ configureDependencies(project)
+ project.plugins.apply(JavaBasePlugin);
+ Task ratTask = project.task("rat",
+ type: RatTask,
+ group: 'Build',
+ description: 'Runs Apache Rat checks.')
+ project.tasks[JavaBasePlugin.CHECK_TASK_NAME].dependsOn ratTask
+ }
+
+ void configureDependencies(final Project project) {
+ project.configurations {
+ rat
+ }
+ project.repositories {
+ mavenCentral()
+ }
+ project.dependencies {
+ rat 'org.apache.rat:apache-rat-tasks:0.8'
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/gradle/resources/rat-output-to-html.xsl
----------------------------------------------------------------------
diff --git a/gradle/resources/rat-output-to-html.xsl b/gradle/resources/rat-output-to-html.xsl
new file mode 100644
index 0000000..97ea7a1
--- /dev/null
+++ b/gradle/resources/rat-output-to-html.xsl
@@ -0,0 +1,206 @@
+<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0">
+
+<!--***********************************************************
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ ***********************************************************-->
+
+<!-- This style sheet converts any rat-report.xml file. -->
+
+<xsl:template match="/">
+
+ <html>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+ <style type="text/css">
+ <!--
+body {margin-top: 0px;font-size: 0.8em;background-color: #F9F7ED;}
+
+h1 {color:red;}
+h2 {color:blue;}
+h3 {color:green;}
+h4 {color:orange;}
+
+/* Table Design */
+
+table,tr,td {text-align:center;font-weight:bold;border:1px solid #000;}
+caption {color:blue;text-align:left;}
+.notes, .binaries, .archives, .standards {width:25%;}
+.notes {background:#D7EDEE;}
+.binaries {background:#D0F2F4;}
+.archives {background:#ABE7E9;}
+.standards {background:#A0F0F4;}
+.licenced, .generated {width:50%;}
+.licenced {background:#C6EBDD;}
+.generated {background:#ABE9D2;}
+.java_note {background:#D6EBC6;}
+.generated_note {background:#C9E7A9;}
+.unknown {width:100%;background:#E92020;}
+.unknown-zero {color:#00CC00;}
+.center{text-align:center;margin:0 auto;}
+-->
+ </style>
+ </head>
+ <body>
+ <xsl:apply-templates/>
+ <xsl:call-template name="generated"/>
+ </body>
+ </html>
+</xsl:template>
+
+<xsl:template match="rat-report">
+
+ <h1>Rat Report</h1>
+ <p>This HTML version (yes, it is!) is generated from the RAT xml reports using Saxon9B. All the outputs required are displayed below, similar to the .txt version.
+ This is obviously a work in progress; and a prettier, easier to read and manage version will be available soon</p>
+<div class="center">
+<table id="rat-reports summary" cellspacing="0" summary="A snapshot summary of this rat report">
+<caption>
+Table 1: A snapshot summary of this rat report.
+</caption>
+ <tr>
+ <td colspan="1" class="notes">Notes: <xsl:value-of select="count(descendant::type[attribute::name="notice"])"/></td>
+ <td colspan="1" class="binaries">Binaries: <xsl:value-of select="count(descendant::type[attribute::name="binary"])"/></td>
+ <td colspan="1" class="archives">Archives: <xsl:value-of select="count(descendant::type[attribute::name="archive"])"/></td>
+ <td colspan="1" class="standards">Standards: <xsl:value-of select="count(descendant::type[attribute::name="standard"])"/></td>
+ </tr>
+ <tr>
+ <td colspan="2" class="licenced">Apache Licensed: <xsl:value-of select="count(descendant::header-type[attribute::name="AL "])"/></td>
+ <td colspan="2" class="generated">Generated Documents: <xsl:value-of select="count(descendant::header-type[attribute::name="GEN "])"/></td>
+ </tr>
+ <tr>
+ <td colspan="2" class="java_note">Note: JavaDocs are generated and so license header is optional</td>
+ <td colspan="2" class="generated_note">Note: Generated files do not require license headers</td>
+ </tr>
+ <tr>
+<xsl:choose>
+ <xsl:when test="count(descendant::header-type[attribute::name="?????"]) > 0">
+ <td colspan="4" class="unknown"><xsl:value-of select="count(descendant::header-type[attribute::name="?????"])"/> Unknown Licenses - or files without a license.</td>
+ </xsl:when>
+ <xsl:otherwise>
+ <td colspan="4" class="unknown-zero"><xsl:value-of select="count(descendant::header-type[attribute::name="?????"])"/> Unknown Licenses - or files without a license.</td>
+ </xsl:otherwise>
+</xsl:choose>
+ </tr>
+</table>
+</div>
+<hr/>
+ <h3>Unapproved Licenses:</h3>
+
+ <xsl:for-each select="descendant::resource[license-approval/@name="false"]">
+ <xsl:text> </xsl:text>
+ <xsl:value-of select="@name"/><br/>
+ <xsl:text>
+</xsl:text>
+</xsl:for-each>
+<hr/>
+
+<h3>Archives:</h3>
+
+<xsl:for-each select="descendant::resource[type/@name="archive"]">
+ + <xsl:value-of select="@name"/>
+ <br/>
+ </xsl:for-each>
+ <hr/>
+
+ <p>
+ Files with Apache License headers will be marked AL<br/>
+ Binary files (which do not require AL headers) will be marked B<br/>
+ Compressed archives will be marked A<br/>
+ Notices, licenses etc will be marked N<br/>
+ </p>
+
+ <xsl:for-each select="descendant::resource">
+ <xsl:choose>
+ <xsl:when test="license-approval/@name="false"">!</xsl:when>
+ <xsl:otherwise><xsl:text> </xsl:text></xsl:otherwise>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="type/@name="notice"">N </xsl:when>
+ <xsl:when test="type/@name="archive"">A </xsl:when>
+ <xsl:when test="type/@name="binary"">B </xsl:when>
+ <xsl:when test="type/@name="standard""><xsl:value-of select="header-type/@name"/></xsl:when>
+ <xsl:otherwise>!!!!!</xsl:otherwise>
+ </xsl:choose>
+ <xsl:text> </xsl:text>
+ <xsl:value-of select="@name"/><br/>
+ <xsl:text>
+ </xsl:text>
+ </xsl:for-each>
+ <hr/>
+
+ <h3>Printing headers for files without AL header...</h3>
+
+ <xsl:for-each select="descendant::resource[header-type/@name="?????"]">
+
+ <h4><xsl:value-of select="@name"/></h4>
+ <xsl:value-of select="header-sample"/>
+ <hr/>
+</xsl:for-each>
+<br/>
+
+ <!-- <xsl:apply-templates select="resource"/>
+ <xsl:apply-templates select="header-sample"/>
+ <xsl:apply-templates select="header-type"/>
+ <xsl:apply-templates select="license-family"/>
+ <xsl:apply-templates select="license-approval"/>
+ <xsl:apply-templates select="type"/> -->
+
+</xsl:template>
+
+<xsl:template match="resource">
+ <div>
+ <h3>Resource: <xsl:value-of select="@name"/></h3>
+ <xsl:apply-templates/>
+ </div>
+</xsl:template>
+
+<xsl:template match="header-sample">
+ <xsl:if test="normalize-space(.) != ''">
+ <h4>First few lines of non-compliant file</h4>
+ <p>
+ <xsl:value-of select="."/>
+ </p>
+ </xsl:if>
+ <h4>Other Info:</h4>
+</xsl:template>
+
+<xsl:template match="header-type">
+ Header Type: <xsl:value-of select="@name"/>
+ <br/>
+</xsl:template>
+
+<xsl:template match="license-family">
+ License Family: <xsl:value-of select="@name"/>
+ <br/>
+</xsl:template>
+
+<xsl:template match="license-approval">
+ License Approval: <xsl:value-of select="@name"/>
+ <br/>
+</xsl:template>
+
+<xsl:template match="type">
+ Type: <xsl:value-of select="@name"/>
+ <br/>
+</xsl:template>
+
+<xsl:template name="generated">
+</xsl:template>
+</xsl:transform>