You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by jd...@apache.org on 2016/02/23 00:45:16 UTC
[2/3] incubator-kudu git commit: Add RAT excludes file and parsing
script for releases
Add RAT excludes file and parsing script for releases
The RAT tool itself doesn't have very robust support for exclude files
(only filenames, not paths, are supported), so I've written a little
Python script to parse the output of the RAT XML report and apply rules
from a simple path-globbing exclude file. Instructions in the README.
Ultimately, it would be nice to fully automate downloading and running
RAT, but this is better than nothing.
Change-Id: Id6850e71665c0846b5f57d0527b930a629f0ab4a
Reviewed-on: http://gerrit.cloudera.org:8080/2180
Tested-by: Kudu Jenkins
Reviewed-by: Jean-Daniel Cryans
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/b3efaf02
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/b3efaf02
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/b3efaf02
Branch: refs/heads/master
Commit: b3efaf029a5259f535cded5b21e9c1003bb75726
Parents: dbc1de3
Author: Mike Percy <mp...@apache.org>
Authored: Wed Feb 17 00:50:01 2016 +0200
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Mon Feb 22 23:43:20 2016 +0000
----------------------------------------------------------------------
build-support/release/README.adoc | 27 ++++
build-support/release/check-rat-report.py | 57 +++++++
build-support/release/rat_exclude_files.txt | 181 +++++++++++++++++++++++
3 files changed, 265 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/b3efaf02/build-support/release/README.adoc
----------------------------------------------------------------------
diff --git a/build-support/release/README.adoc b/build-support/release/README.adoc
new file mode 100644
index 0000000..b54f0ef
--- /dev/null
+++ b/build-support/release/README.adoc
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+= How to run RAT on a release tarball
+
+Kudu has a script and an excludes file to allow for running RAT on a release
+artifact. You can use the following steps to verify a source tarball.
+
+1. Download a binary artifact of RAT from
+ http://creadur.apache.org/rat/download_rat.cgi and unpack it.
+2. Run RAT: `java -jar apache-rat-0.11.jar -x apache-kudu-X.Y.Z.tar.gz > rat-report.xml`
+3. ./build-support/release/check-rat-report.py build-support/release/rat_exclude_files.txt rat-report.xml
+
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/b3efaf02/build-support/release/check-rat-report.py
----------------------------------------------------------------------
diff --git a/build-support/release/check-rat-report.py b/build-support/release/check-rat-report.py
new file mode 100755
index 0000000..213fd9b
--- /dev/null
+++ b/build-support/release/check-rat-report.py
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+##############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##############################################################################
+import fnmatch
+import re
+import sys
+import xml.etree.ElementTree as ET
+
+if len(sys.argv) != 3:
+ sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" % (sys.argv[0],))
+ sys.exit(1)
+
+exclude_globs_filename = sys.argv[1]
+xml_filename = sys.argv[2]
+
+globs = [line.strip() for line in open(exclude_globs_filename, "r")]
+
+tree = ET.parse(xml_filename)
+root = tree.getroot()
+resources = root.findall('resource')
+
+all_ok = True
+for r in resources:
+ approvals = r.findall('license-approval')
+ if not approvals or approvals[0].attrib['name'] == 'true':
+ continue
+ clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
+ excluded = False
+ for g in globs:
+ if fnmatch.fnmatch(clean_name, g):
+ excluded = True
+ break
+ if not excluded:
+ sys.stderr.write("NOT APPROVED: %s (%s): %s\n" % (clean_name, r.attrib['name'], approvals[0].attrib['name']))
+ all_ok = False
+
+if not all_ok:
+ sys.exit(1)
+
+print 'OK'
+sys.exit(0)
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/b3efaf02/build-support/release/rat_exclude_files.txt
----------------------------------------------------------------------
diff --git a/build-support/release/rat_exclude_files.txt b/build-support/release/rat_exclude_files.txt
new file mode 100644
index 0000000..a70d6aa
--- /dev/null
+++ b/build-support/release/rat_exclude_files.txt
@@ -0,0 +1,181 @@
+pax_global_header
+.gitignore
+*/.gitignore
+.ycm_extra_conf.py
+*.json
+*.graffle
+*/Gemfile.lock
+*.bib
+*.pdf
+version.txt
+build-support/release/rat_exclude_files.txt
+docs/whitepaper/vldb.cls
+java/kudu-client/src/main/java/com/google/protobuf/ZeroCopyLiteralByteString.java
+java/kudu-client/src/main/java/org/kududb/client/AsyncKuduClient.java
+java/kudu-client/src/main/java/org/kududb/client/AsyncKuduScanner.java
+java/kudu-client/src/main/java/org/kududb/client/Bytes.java
+java/kudu-client/src/main/java/org/kududb/client/ConnectionResetException.java
+java/kudu-client/src/main/java/org/kududb/client/HasFailedRpcException.java
+java/kudu-client/src/main/java/org/kududb/client/InvalidResponseException.java
+java/kudu-client/src/main/java/org/kududb/client/KuduException.java
+java/kudu-client/src/main/java/org/kududb/client/KuduRpc.java
+java/kudu-client/src/main/java/org/kududb/client/NonRecoverableException.java
+java/kudu-client/src/main/java/org/kududb/client/PleaseThrottleException.java
+java/kudu-client/src/main/java/org/kududb/client/RecoverableException.java
+java/kudu-client/src/main/java/org/kududb/client/SecureRpcHelper.java
+java/kudu-client/src/main/java/org/kududb/client/TabletClient.java
+java/kudu-client/src/test/resources/flags
+python/MANIFEST.in
+python/README.md
+python/requirements.txt
+python/setup.cfg
+src/kudu/client/symbols.map
+src/kudu/gutil/arm_instruction_set_select.h
+src/kudu/gutil/atomic_refcount.h
+src/kudu/gutil/atomicops-internals-macosx.h
+src/kudu/gutil/atomicops-internals-tsan.h
+src/kudu/gutil/atomicops.h
+src/kudu/gutil/basictypes.h
+src/kudu/gutil/bind.h
+src/kudu/gutil/bind.h.pump
+src/kudu/gutil/bind_helpers.h
+src/kudu/gutil/bind_internal.h
+src/kudu/gutil/bind_internal.h.pump
+src/kudu/gutil/bits.cc
+src/kudu/gutil/bits.h
+src/kudu/gutil/callback.h
+src/kudu/gutil/callback.h.pump
+src/kudu/gutil/callback_forward.h
+src/kudu/gutil/callback_internal.cc
+src/kudu/gutil/callback_internal.h
+src/kudu/gutil/casts.h
+src/kudu/gutil/charmap.h
+src/kudu/gutil/cpu.cc
+src/kudu/gutil/cpu.h
+src/kudu/gutil/dynamic_annotations.c
+src/kudu/gutil/dynamic_annotations.h
+src/kudu/gutil/gscoped_ptr.h
+src/kudu/gutil/int128.cc
+src/kudu/gutil/int128.h
+src/kudu/gutil/integral_types.h
+src/kudu/gutil/linux_syscall_support.h
+src/kudu/gutil/macros.h
+src/kudu/gutil/manual_constructor.h
+src/kudu/gutil/map-util.h
+src/kudu/gutil/move.h
+src/kudu/gutil/once.cc
+src/kudu/gutil/once.h
+src/kudu/gutil/paranoid.h
+src/kudu/gutil/port.h
+src/kudu/gutil/raw_scoped_refptr_mismatch_checker.h
+src/kudu/gutil/ref_counted.cc
+src/kudu/gutil/ref_counted.h
+src/kudu/gutil/ref_counted_memory.cc
+src/kudu/gutil/ref_counted_memory.h
+src/kudu/gutil/singleton.h
+src/kudu/gutil/spinlock.cc
+src/kudu/gutil/spinlock.h
+src/kudu/gutil/spinlock_internal.cc
+src/kudu/gutil/spinlock_internal.h
+src/kudu/gutil/spinlock_linux-inl.h
+src/kudu/gutil/spinlock_posix-inl.h
+src/kudu/gutil/spinlock_win32-inl.h
+src/kudu/gutil/stringprintf.cc
+src/kudu/gutil/stringprintf.h
+src/kudu/gutil/strtoint.cc
+src/kudu/gutil/strtoint.h
+src/kudu/gutil/synchronization_profiling.h
+src/kudu/gutil/sysinfo.cc
+src/kudu/gutil/sysinfo.h
+src/kudu/gutil/template_util.h
+src/kudu/gutil/thread_annotations.h
+src/kudu/gutil/tuple.h
+src/kudu/gutil/type_traits.h
+src/kudu/gutil/valgrind.h
+src/kudu/gutil/auxiliary/atomicops-internals-arm-generic.h
+src/kudu/gutil/auxiliary/atomicops-internals-arm-v6plus.h
+src/kudu/gutil/auxiliary/atomicops-internals-windows.h
+src/kudu/gutil/hash/builtin_type_hash.h
+src/kudu/gutil/hash/city.cc
+src/kudu/gutil/hash/city.h
+src/kudu/gutil/hash/hash.cc
+src/kudu/gutil/hash/hash.h
+src/kudu/gutil/hash/hash128to64.h
+src/kudu/gutil/hash/jenkins.cc
+src/kudu/gutil/hash/jenkins.h
+src/kudu/gutil/hash/jenkins_lookup2.h
+src/kudu/gutil/hash/legacy_hash.h
+src/kudu/gutil/hash/string_hash.h
+src/kudu/gutil/strings/ascii_ctype.cc
+src/kudu/gutil/strings/ascii_ctype.h
+src/kudu/gutil/strings/charset.cc
+src/kudu/gutil/strings/charset.h
+src/kudu/gutil/strings/escaping.cc
+src/kudu/gutil/strings/escaping.h
+src/kudu/gutil/strings/fastmem.h
+src/kudu/gutil/strings/human_readable.cc
+src/kudu/gutil/strings/human_readable.h
+src/kudu/gutil/strings/join.cc
+src/kudu/gutil/strings/join.h
+src/kudu/gutil/strings/memutil.cc
+src/kudu/gutil/strings/memutil.h
+src/kudu/gutil/strings/numbers.cc
+src/kudu/gutil/strings/numbers.h
+src/kudu/gutil/strings/serialize.cc
+src/kudu/gutil/strings/serialize.h
+src/kudu/gutil/strings/split.cc
+src/kudu/gutil/strings/split.h
+src/kudu/gutil/strings/split_internal.h
+src/kudu/gutil/strings/strcat.cc
+src/kudu/gutil/strings/strcat.h
+src/kudu/gutil/strings/stringpiece.cc
+src/kudu/gutil/strings/stringpiece.h
+src/kudu/gutil/strings/strip.cc
+src/kudu/gutil/strings/strip.h
+src/kudu/gutil/strings/substitute.cc
+src/kudu/gutil/strings/substitute.h
+src/kudu/gutil/strings/util.cc
+src/kudu/gutil/strings/util.h
+src/kudu/gutil/threading/thread_collision_warner.cc
+src/kudu/gutil/threading/thread_collision_warner.h
+src/kudu/gutil/utf/rune.c
+src/kudu/gutil/utf/utf.h
+src/kudu/gutil/utf/utfdef.h
+src/kudu/twitter-demo/example-deletes.txt
+src/kudu/twitter-demo/example-tweets.txt
+src/kudu/util/cache.h
+src/kudu/util/coding.cc
+src/kudu/util/coding.h
+src/kudu/util/condition_variable.cc
+src/kudu/util/condition_variable.h
+src/kudu/util/env.cc
+src/kudu/util/env.h
+src/kudu/util/env_posix.cc
+src/kudu/util/random.h
+src/kudu/util/slice.h
+src/kudu/util/status.cc
+src/kudu/util/status.h
+src/kudu/util/debug/trace_event.h
+src/kudu/util/debug/trace_event_impl.cc
+src/kudu/util/debug/trace_event_impl.h
+src/kudu/util/debug/trace_event_impl_constants.cc
+src/kudu/util/debug/trace_event_synthetic_delay.cc
+src/kudu/util/debug/trace_event_synthetic_delay.h
+src/kudu/util/memenv/memenv-test.cc
+src/kudu/util/memenv/memenv.cc
+src/kudu/util/memenv/memenv.h
+thirdparty/boost_uuid/boost/uuid/name_generator.hpp
+thirdparty/boost_uuid/boost/uuid/nil_generator.hpp
+thirdparty/boost_uuid/boost/uuid/random_generator.hpp
+thirdparty/boost_uuid/boost/uuid/seed_rng.hpp
+thirdparty/boost_uuid/boost/uuid/sha1.hpp
+thirdparty/boost_uuid/boost/uuid/string_generator.hpp
+thirdparty/boost_uuid/boost/uuid/uuid.hpp
+thirdparty/boost_uuid/boost/uuid/uuid_generators.hpp
+thirdparty/boost_uuid/boost/uuid/uuid_io.hpp
+thirdparty/boost_uuid/boost/uuid/uuid_serialize.hpp
+thirdparty/patches/*.patch
+www/d3.v2.js
+www/epoch.0.5.2.min.css
+www/epoch.0.5.2.min.js
+www/jquery-1.11.1.min.js