You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2018/02/10 19:25:51 UTC
lucene-solr:branch_7x: LUCENE-8168: Move Groovy scripts in build
files to separate files. Update Groovy to 2.4.13
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 5dd5a5aae -> f2c4ca5ed
LUCENE-8168: Move Groovy scripts in build files to separate files. Update Groovy to 2.4.13
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f2c4ca5e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f2c4ca5e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f2c4ca5e
Branch: refs/heads/branch_7x
Commit: f2c4ca5ed3c6b46a006d5bf0ec45c2a17c25855a
Parents: 5dd5a5a
Author: Uwe Schindler <us...@apache.org>
Authored: Sat Feb 10 20:24:33 2018 +0100
Committer: Uwe Schindler <us...@apache.org>
Committed: Sat Feb 10 20:25:33 2018 +0100
----------------------------------------------------------------------
build.xml | 241 +------------------
lucene/CHANGES.txt | 5 +
lucene/common-build.xml | 100 +-------
.../src/groovy/check-source-patterns.groovy | 189 +++++++++++++++
.../tools/src/groovy/check-working-copy.groovy | 61 +++++
.../src/groovy/install-markdown-filter.groovy | 61 +++++
.../tools/src/groovy/patch-mrjar-classes.groovy | 5 +
lucene/tools/src/groovy/run-beaster.groovy | 71 ++++++
lucene/tools/src/groovy/run-maven-build.groovy | 49 ++++
9 files changed, 448 insertions(+), 334 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f2c4ca5e/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 2346d54..b5d3015 100755
--- a/build.xml
+++ b/build.xml
@@ -125,177 +125,7 @@
</target>
<target name="-validate-source-patterns" unless="disable.source-patterns" depends="resolve-groovy,rat-sources-typedef">
- <!-- check that there are no @author javadoc tags, tabs, svn keywords, javadoc-style licenses, or nocommits: -->
- <property name="validate.baseDir" location="."/>
- <groovy taskname="source-patterns" classpathref="rat.classpath"><![CDATA[
- import org.apache.tools.ant.BuildException;
- import org.apache.tools.ant.Project;
- import org.apache.rat.Defaults;
- import org.apache.rat.document.impl.FileDocument;
- import org.apache.rat.api.MetaData;
-
- def extensions = [
- 'java', 'jflex', 'py', 'pl', 'g4', 'jj', 'html', 'js',
- 'css', 'xml', 'xsl', 'vm', 'sh', 'cmd', 'bat', 'policy',
- 'properties', 'mdtext',
- 'template', 'adoc', 'json',
- ];
- def invalidPatterns = [
- (~$/@author\b/$) : '@author javadoc tag',
- (~$/(?i)\bno(n|)commit\b/$) : 'nocommit',
- (~$/\bTOOD:/$) : 'TOOD instead TODO',
- (~$/\t/$) : 'tabs instead spaces',
- (~$/\Q/**\E((?:\s)|(?:\*))*\Q{@inheritDoc}\E((?:\s)|(?:\*))*\Q*/\E/$) : '{@inheritDoc} on its own is unnecessary',
- (~$/\$$(?:LastChanged)?Date\b/$) : 'svn keyword',
- (~$/\$$(?:(?:LastChanged)?Revision|Rev)\b/$) : 'svn keyword',
- (~$/\$$(?:LastChangedBy|Author)\b/$) : 'svn keyword',
- (~$/\$$(?:Head)?URL\b/$) : 'svn keyword',
- (~$/\$$Id\b/$) : 'svn keyword',
- (~$/\$$Header\b/$) : 'svn keyword',
- (~$/\$$Source\b/$) : 'svn keyword',
- (~$/^\uFEFF/$) : 'UTF-8 byte order mark'
- ];
-
- def baseDir = properties['validate.baseDir'];
- def baseDirLen = baseDir.length() + 1;
-
- def found = 0;
- def violations = new TreeSet();
- def reportViolation = { f, name ->
- task.log(name + ': ' + f.toString().substring(baseDirLen).replace(File.separatorChar, (char)'/'), Project.MSG_ERR);
- violations.add(name);
- found++;
- }
-
- def javadocsPattern = ~$/(?sm)^\Q/**\E(.*?)\Q*/\E/$;
- def javaCommentPattern = ~$/(?sm)^\Q/*\E(.*?)\Q*/\E/$;
- def xmlCommentPattern = ~$/(?sm)\Q<!--\E(.*?)\Q-->\E/$;
- def lineSplitter = ~$/[\r\n]+/$;
- def singleLineSplitter = ~$/\n\r?/$;
- def licenseMatcher = Defaults.createDefaultMatcher();
- def validLoggerPattern = ~$/(?s)\b(private\s|static\s|final\s){3}+\s*Logger\s+\p{javaJavaIdentifierStart}+\s+=\s+\QLoggerFactory.getLogger(MethodHandles.lookup().lookupClass());\E/$;
- def packagePattern = ~$/(?m)^\s*package\s+org\.apache.*;/$;
- def xmlTagPattern = ~$/(?m)\s*<[a-zA-Z].*/$;
- def sourceHeaderPattern = ~$/\[source\b.*/$;
- def blockBoundaryPattern = ~$/----\s*/$;
- def blockTitlePattern = ~$/\..*/$;
- def unescapedSymbolPattern = ~$/(?<=[^\\]|^)([-=]>|<[-=])/$; // SOLR-10883
-
- def isLicense = { matcher, ratDocument ->
- licenseMatcher.reset();
- return lineSplitter.split(matcher.group(1)).any{ licenseMatcher.match(ratDocument, it) };
- }
-
- def checkLicenseHeaderPrecedes = { f, description, contentPattern, commentPattern, text, ratDocument ->
- def contentMatcher = contentPattern.matcher(text);
- if (contentMatcher.find()) {
- def contentStartPos = contentMatcher.start();
- def commentMatcher = commentPattern.matcher(text);
- while (commentMatcher.find()) {
- if (isLicense(commentMatcher, ratDocument)) {
- if (commentMatcher.start() < contentStartPos) {
- break; // This file is all good, so break loop: license header precedes 'description' definition
- } else {
- reportViolation(f, description+' declaration precedes license header');
- }
- }
- }
- }
- }
-
- def checkMockitoAssume = { f, text ->
- if (text.contains("mockito") && !text.contains("assumeWorkingMockito()")) {
- reportViolation(f, 'File uses Mockito but has no assumeWorkingMockito() call');
- }
- }
-
- def checkForUnescapedSymbolSubstitutions = { f, text ->
- def inCodeBlock = false;
- def underSourceHeader = false;
- def lineNumber = 0;
- singleLineSplitter.split(text).each {
- ++lineNumber;
- if (underSourceHeader) { // This line is either a single source line, or the boundary of a code block
- inCodeBlock = blockBoundaryPattern.matcher(it).matches();
- if ( ! blockTitlePattern.matcher(it).matches()) {
- underSourceHeader = false;
- }
- } else {
- if (inCodeBlock) {
- inCodeBlock = ! blockBoundaryPattern.matcher(it).matches();
- } else {
- underSourceHeader = sourceHeaderPattern.matcher(it).lookingAt();
- if ( ! underSourceHeader) {
- def unescapedSymbolMatcher = unescapedSymbolPattern.matcher(it);
- if (unescapedSymbolMatcher.find()) {
- reportViolation(f, 'Unescaped symbol "' + unescapedSymbolMatcher.group(1) + '" on line #' + lineNumber);
- }
- }
- }
- }
- }
- }
-
- ant.fileScanner{
- fileset(dir: baseDir){
- extensions.each{
- include(name: 'lucene/**/*.' + it)
- include(name: 'solr/**/*.' + it)
- include(name: 'dev-tools/**/*.' + it)
- include(name: '*.' + it)
- }
- // TODO: For now we don't scan txt files, so we
- // check licenses in top-level folders separately:
- include(name: '*.txt')
- include(name: '*/*.txt')
- // excludes:
- exclude(name: '**/build/**')
- exclude(name: '**/dist/**')
- exclude(name: 'lucene/benchmark/work/**')
- exclude(name: 'lucene/benchmark/temp/**')
- exclude(name: '**/CheckLoggingConfiguration.java')
- exclude(name: 'build.xml') // ourselves :-)
- }
- }.each{ f ->
- task.log('Scanning file: ' + f, Project.MSG_VERBOSE);
- def text = f.getText('UTF-8');
- invalidPatterns.each{ pattern,name ->
- if (pattern.matcher(text).find()) {
- reportViolation(f, name);
- }
- }
- def javadocsMatcher = javadocsPattern.matcher(text);
- def ratDocument = new FileDocument(f);
- while (javadocsMatcher.find()) {
- if (isLicense(javadocsMatcher, ratDocument)) {
- reportViolation(f, String.format(Locale.ENGLISH, 'javadoc-style license header [%s]',
- ratDocument.getMetaData().value(MetaData.RAT_URL_LICENSE_FAMILY_NAME)));
- }
- }
- if (f.name.endsWith('.java')) {
- if (text.contains('org.slf4j.LoggerFactory')) {
- if (!validLoggerPattern.matcher(text).find()) {
- reportViolation(f, 'invalid logging pattern [not private static final, uses static class name]');
- }
- }
- checkLicenseHeaderPrecedes(f, 'package', packagePattern, javaCommentPattern, text, ratDocument);
- if (f.name.contains("Test")) {
- checkMockitoAssume(f, text);
- }
- }
- if (f.name.endsWith('.xml') || f.name.endsWith('.xml.template')) {
- checkLicenseHeaderPrecedes(f, '<tag>', xmlTagPattern, xmlCommentPattern, text, ratDocument);
- }
- if (f.name.endsWith('.adoc')) {
- checkForUnescapedSymbolSubstitutions(f, text);
- }
- };
-
- if (found) {
- throw new BuildException(String.format(Locale.ENGLISH, 'Found %d violations in source files (%s).',
- found, violations.join(', ')));
- }
- ]]></groovy>
+ <groovy taskname="source-patterns" classpathref="rat.classpath" src="${common.dir}/tools/src/groovy/check-source-patterns.groovy"/>
</target>
<target name="rat-sources" description="Runs rat across all sources and tests" depends="common.rat-sources">
@@ -410,31 +240,7 @@
</target>
<target name="run-maven-build" depends="get-maven-poms,install-maven-tasks,resolve-groovy" description="Runs the Maven build using automatically generated POMs">
- <groovy><![CDATA[
- import groovy.xml.NamespaceBuilder;
- import org.apache.tools.ant.Project;
- def userHome = properties['user.home'], commonDir = properties['common.dir'];
- def propPrefix = '-mvn.inject.'; int propPrefixLen = propPrefix.length();
- def subProject = project.createSubProject();
- project.copyUserProperties(subProject);
- subProject.initProperties();
- new AntBuilder(subProject).sequential{
- property(file: userHome+'/lucene.build.properties', prefix: propPrefix);
- property(file: userHome+'/build.properties', prefix: propPrefix);
- property(file: commonDir+'/build.properties', prefix: propPrefix);
- };
- def cmdlineProps = subProject.properties
- .findAll{ k, v -> k.startsWith(propPrefix) }
- .collectEntries{ k, v -> [k.substring(propPrefixLen), v] };
- cmdlineProps << project.userProperties.findAll{ k, v -> !k.startsWith('ant.') };
- def artifact = NamespaceBuilder.newInstance(ant, 'antlib:org.apache.maven.artifact.ant');
- task.log('Running Maven with props: ' + cmdlineProps.toString(), Project.MSG_INFO);
- artifact.mvn(pom: properties['maven-build-dir']+'/pom.xml', mavenVersion: properties['maven-version'], failonerror: true, fork: true) {
- cmdlineProps.each{ k, v -> arg(value: '-D' + k + '=' + v) };
- arg(value: '-fae');
- arg(value: 'install');
- };
- ]]></groovy>
+ <groovy src="${common.dir}/tools/src/groovy/run-maven-build.groovy"/>
</target>
<target name="remove-maven-artifacts" description="Removes all Lucene/Solr Maven artifacts from the local repository">
@@ -667,48 +473,7 @@ File | Project Structure | Platform Settings | SDKs):
<ivy:cachepath xmlns:ivy="antlib:org.apache.ivy.ant"
organisation="org.eclipse.jgit" module="org.eclipse.jgit" revision="${jgit-version}"
inline="true" conf="default" transitive="true" pathid="jgit.classpath"/>
- <groovy taskname="wc-checker" classpathref="jgit.classpath"><![CDATA[
- import org.apache.tools.ant.BuildException;
- import org.apache.tools.ant.Project;
- import org.eclipse.jgit.api.Git;
- import org.eclipse.jgit.api.Status;
- import org.eclipse.jgit.lib.Repository;
- import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
- import org.eclipse.jgit.errors.*;
-
- def setProjectPropertyFromSet(prop, set) {
- if (set) {
- properties[prop] = '* ' + set.join(properties['line.separator'] + '* ');
- }
- };
-
- try {
- task.log('Initializing working copy...', Project.MSG_INFO);
- final Repository repository = new FileRepositoryBuilder()
- .setWorkTree(project.getBaseDir())
- .setMustExist(true)
- .build();
-
- task.log('Checking working copy status...', Project.MSG_INFO);
- final Status status = new Git(repository).status().call();
- if (!status.isClean()) {
- final SortedSet unversioned = new TreeSet(), modified = new TreeSet();
- status.properties.each{ prop, val ->
- if (val instanceof Set) {
- if (prop in ['untracked', 'untrackedFolders', 'missing']) {
- unversioned.addAll(val);
- } else if (prop != 'ignoredNotInIndex') {
- modified.addAll(val);
- }
- }
- };
- setProjectPropertyFromSet('wc.unversioned.files', unversioned);
- setProjectPropertyFromSet('wc.modified.files', modified);
- }
- } catch (RepositoryNotFoundException | NoWorkTreeException | NotSupportedException e) {
- task.log('WARNING: Development directory is not a valid GIT checkout! Disabling checks...', Project.MSG_WARN);
- }
- ]]></groovy>
+ <groovy taskname="wc-checker" classpathref="jgit.classpath" src="${common.dir}/tools/src/groovy/check-working-copy.groovy"/>
<fail if="wc.unversioned.files"
message="Source checkout is dirty (unversioned/missing files) after running tests!!! Offending files:${line.separator}${wc.unversioned.files}"/>
<fail message="Source checkout is modified!!! Offending files:${line.separator}${wc.modified.files}">
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f2c4ca5e/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 040dffe..1e5ea3d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -108,6 +108,11 @@ Other
* LUCENE-8155: Add back support in smoke tester to run against later Java versions.
(Uwe Schindler)
+Build
+
+* LUCENE-8168: Moved Groovy scripts in build files to separate files.
+ Update Groovy to 2.4.13. (Uwe Schindler)
+
======================= Lucene 7.2.1 =======================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f2c4ca5e/lucene/common-build.xml
----------------------------------------------------------------------
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index e63b116..539d206 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -568,7 +568,7 @@
<loadproperties prefix="ivyversions" srcFile="${common.dir}/ivy-versions.properties"/>
<ivy:cachepath organisation="org.ow2.asm" module="asm-commons" revision="${ivyversions./org.ow2.asm/asm-commons}"
inline="true" conf="default" transitive="true" log="download-only" pathid="asm.classpath"/>
- <groovy classpathref="asm.classpath" src="${common.dir}/tools/src/groovy/patch-mrjar-classes.groovy"/>
+ <groovy taskname="patch-cls" classpathref="asm.classpath" src="${common.dir}/tools/src/groovy/patch-mrjar-classes.groovy"/>
<touch file="${build.dir}/patch-mrjar.stamp"/>
</target>
@@ -1568,58 +1568,7 @@ ${tests-output}/junit4-*.suites - per-JVM executed suites
<not><isreference refid="junit.classpath"/></not>
</condition>
</fail>
- <groovy taskname="beaster"><![CDATA[
- import org.apache.tools.ant.BuildException;
- import org.apache.tools.ant.BuildLogger;
- import org.apache.tools.ant.Project;
-
- int iters = (properties['beast.iters'] ?: '1') as int;
- if (iters <= 1) {
- throw new BuildException("Please give -Dbeast.iters with an int value > 1.");
- }
-
- def antcall = project.createTask('antcall');
- antcall.with {
- target = '-test';
- inheritAll = true;
- inheritRefs = true;
- createParam().with {
- name = "tests.isbeasting";
- value = "true";
- };
- };
-
- (1..iters).each { i ->
- task.log('Beast round: ' + i, Project.MSG_INFO);
- try {
- // disable verbose build logging:
- project.buildListeners.each { listener ->
- if (listener instanceof BuildLogger) {
- listener.messageOutputLevel = Project.MSG_WARN;
- }
- };
-
- antcall.execute();
-
- } catch (BuildException be) {
- def logFile = new File(properties["junit.output.dir"], "tests-failures.txt");
- if (logFile.exists()) {
- logFile.eachLine("UTF-8", { line ->
- task.log(line, Project.MSG_ERR);
- });
- }
- throw be;
- } finally {
- // restore build logging (unfortunately there is no way to get the original logging level (write-only property):
- project.buildListeners.each { listener ->
- if (listener instanceof BuildLogger) {
- listener.messageOutputLevel = Project.MSG_INFO;
- }
- };
- }
- };
- task.log('Beasting finished.', Project.MSG_INFO);
- ]]></groovy>
+ <groovy taskname="beaster" src="${common.dir}/tools/src/groovy/run-beaster.groovy"/>
</target>
<target name="-check-totals" if="tests.totals.toplevel" depends="resolve-groovy">
@@ -2484,7 +2433,7 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
<!-- GROOVY scripting engine for ANT tasks -->
<target name="resolve-groovy" unless="groovy.loaded" depends="ivy-availability-check,ivy-configure">
- <ivy:cachepath organisation="org.codehaus.groovy" module="groovy-all" revision="2.4.12"
+ <ivy:cachepath organisation="org.codehaus.groovy" module="groovy-all" revision="2.4.13"
inline="true" conf="default" type="jar" transitive="true" pathid="groovy.classpath"/>
<taskdef name="groovy"
classname="org.codehaus.groovy.ant.Groovy"
@@ -2567,48 +2516,7 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
<ivy:dependency org="com.vladsch.flexmark" name="flexmark-ext-autolink" rev="${flexmark.version}" conf="default" />
<ivy:dependency org="com.vladsch.flexmark" name="flexmark-ext-abbreviation" rev="${flexmark.version}" conf="default" />
</ivy:cachepath>
- <groovy classpathref="markdown.classpath"><![CDATA[
- import org.apache.tools.ant.AntTypeDefinition;
- import org.apache.tools.ant.ComponentHelper;
- import org.apache.tools.ant.filters.TokenFilter.ChainableReaderFilter;
- import com.vladsch.flexmark.ast.Node;
- import com.vladsch.flexmark.ast.Heading;
- import com.vladsch.flexmark.html.HtmlRenderer;
- import com.vladsch.flexmark.parser.Parser;
- import com.vladsch.flexmark.parser.ParserEmulationProfile;
- import com.vladsch.flexmark.util.html.Escaping;
- import com.vladsch.flexmark.util.options.MutableDataSet;
- import com.vladsch.flexmark.ext.abbreviation.AbbreviationExtension;
- import com.vladsch.flexmark.ext.autolink.AutolinkExtension;
-
- public final class MarkdownFilter extends ChainableReaderFilter {
- @Override
- public String filter(String markdownSource) {
- MutableDataSet options = new MutableDataSet();
- options.setFrom(ParserEmulationProfile.MARKDOWN);
- options.set(Parser.EXTENSIONS, [ AbbreviationExtension.create(), AutolinkExtension.create() ]);
- options.set(HtmlRenderer.RENDER_HEADER_ID, true);
- options.set(HtmlRenderer.MAX_TRAILING_BLANK_LINES, 0);
- Node parsed = Parser.builder(options).build().parse(markdownSource);
-
- StringBuilder html = new StringBuilder('<html>\n<head>\n');
- CharSequence title = parsed.getFirstChildAny(Heading.class)?.getText();
- if (title != null) {
- html.append('<title>').append(Escaping.escapeHtml(title, false)).append('</title>\n');
- }
- html.append('<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">\n')
- .append('</head>\n<body>\n');
- HtmlRenderer.builder(options).build().render(parsed, html);
- html.append('</body>\n</html>\n');
- return html;
- }
- }
-
- AntTypeDefinition t = new AntTypeDefinition();
- t.setName('markdownfilter');
- t.setClass(MarkdownFilter.class);
- ComponentHelper.getComponentHelper(project).addDataTypeDefinition(t);
- ]]></groovy>
+ <groovy classpathref="markdown.classpath" src="${common.dir}/tools/src/groovy/install-markdown-filter.groovy"/>
<property name="markdown.loaded" value="true"/>
</target>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f2c4ca5e/lucene/tools/src/groovy/check-source-patterns.groovy
----------------------------------------------------------------------
diff --git a/lucene/tools/src/groovy/check-source-patterns.groovy b/lucene/tools/src/groovy/check-source-patterns.groovy
new file mode 100644
index 0000000..f0007f5
--- /dev/null
+++ b/lucene/tools/src/groovy/check-source-patterns.groovy
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Task script that is called by Ant's build.xml file:
+ * Checks that there are no @author javadoc tags, tabs,
+ * svn keywords, javadoc-style licenses, or nocommits.
+ */
+
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.Project;
+import org.apache.rat.Defaults;
+import org.apache.rat.document.impl.FileDocument;
+import org.apache.rat.api.MetaData;
+
+def extensions = [
+ 'java', 'jflex', 'py', 'pl', 'g4', 'jj', 'html', 'js',
+ 'css', 'xml', 'xsl', 'vm', 'sh', 'cmd', 'bat', 'policy',
+ 'properties', 'mdtext', 'groovy',
+ 'template', 'adoc', 'json',
+];
+def invalidPatterns = [
+ (~$/@author\b/$) : '@author javadoc tag',
+ (~$/(?i)\bno(n|)commit\b/$) : 'nocommit',
+ (~$/\bTOOD:/$) : 'TOOD instead TODO',
+ (~$/\t/$) : 'tabs instead spaces',
+ (~$/\Q/**\E((?:\s)|(?:\*))*\Q{@inheritDoc}\E((?:\s)|(?:\*))*\Q*/\E/$) : '{@inheritDoc} on its own is unnecessary',
+ (~$/\$$(?:LastChanged)?Date\b/$) : 'svn keyword',
+ (~$/\$$(?:(?:LastChanged)?Revision|Rev)\b/$) : 'svn keyword',
+ (~$/\$$(?:LastChangedBy|Author)\b/$) : 'svn keyword',
+ (~$/\$$(?:Head)?URL\b/$) : 'svn keyword',
+ (~$/\$$Id\b/$) : 'svn keyword',
+ (~$/\$$Header\b/$) : 'svn keyword',
+ (~$/\$$Source\b/$) : 'svn keyword',
+ (~$/^\uFEFF/$) : 'UTF-8 byte order mark'
+];
+
+def baseDir = properties['basedir'];
+def baseDirLen = baseDir.length() + 1;
+
+def found = 0;
+def violations = new TreeSet();
+def reportViolation = { f, name ->
+ task.log(name + ': ' + f.toString().substring(baseDirLen).replace(File.separatorChar, (char)'/'), Project.MSG_ERR);
+ violations.add(name);
+ found++;
+}
+
+def javadocsPattern = ~$/(?sm)^\Q/**\E(.*?)\Q*/\E/$;
+def javaCommentPattern = ~$/(?sm)^\Q/*\E(.*?)\Q*/\E/$;
+def xmlCommentPattern = ~$/(?sm)\Q<!--\E(.*?)\Q-->\E/$;
+def lineSplitter = ~$/[\r\n]+/$;
+def singleLineSplitter = ~$/\n\r?/$;
+def licenseMatcher = Defaults.createDefaultMatcher();
+def validLoggerPattern = ~$/(?s)\b(private\s|static\s|final\s){3}+\s*Logger\s+\p{javaJavaIdentifierStart}+\s+=\s+\QLoggerFactory.getLogger(MethodHandles.lookup().lookupClass());\E/$;
+def packagePattern = ~$/(?m)^\s*package\s+org\.apache.*;/$;
+def xmlTagPattern = ~$/(?m)\s*<[a-zA-Z].*/$;
+def sourceHeaderPattern = ~$/\[source\b.*/$;
+def blockBoundaryPattern = ~$/----\s*/$;
+def blockTitlePattern = ~$/\..*/$;
+def unescapedSymbolPattern = ~$/(?<=[^\\]|^)([-=]>|<[-=])/$; // SOLR-10883
+
+def isLicense = { matcher, ratDocument ->
+ licenseMatcher.reset();
+ return lineSplitter.split(matcher.group(1)).any{ licenseMatcher.match(ratDocument, it) };
+}
+
+def checkLicenseHeaderPrecedes = { f, description, contentPattern, commentPattern, text, ratDocument ->
+ def contentMatcher = contentPattern.matcher(text);
+ if (contentMatcher.find()) {
+ def contentStartPos = contentMatcher.start();
+ def commentMatcher = commentPattern.matcher(text);
+ while (commentMatcher.find()) {
+ if (isLicense(commentMatcher, ratDocument)) {
+ if (commentMatcher.start() < contentStartPos) {
+ break; // This file is all good, so break loop: license header precedes 'description' definition
+ } else {
+ reportViolation(f, description+' declaration precedes license header');
+ }
+ }
+ }
+ }
+}
+
+def checkMockitoAssume = { f, text ->
+ if (text.contains("mockito") && !text.contains("assumeWorkingMockito()")) {
+ reportViolation(f, 'File uses Mockito but has no assumeWorkingMockito() call');
+ }
+}
+
+def checkForUnescapedSymbolSubstitutions = { f, text ->
+ def inCodeBlock = false;
+ def underSourceHeader = false;
+ def lineNumber = 0;
+ singleLineSplitter.split(text).each {
+ ++lineNumber;
+ if (underSourceHeader) { // This line is either a single source line, or the boundary of a code block
+ inCodeBlock = blockBoundaryPattern.matcher(it).matches();
+ if ( ! blockTitlePattern.matcher(it).matches()) {
+ underSourceHeader = false;
+ }
+ } else {
+ if (inCodeBlock) {
+ inCodeBlock = ! blockBoundaryPattern.matcher(it).matches();
+ } else {
+ underSourceHeader = sourceHeaderPattern.matcher(it).lookingAt();
+ if ( ! underSourceHeader) {
+ def unescapedSymbolMatcher = unescapedSymbolPattern.matcher(it);
+ if (unescapedSymbolMatcher.find()) {
+ reportViolation(f, 'Unescaped symbol "' + unescapedSymbolMatcher.group(1) + '" on line #' + lineNumber);
+ }
+ }
+ }
+ }
+ }
+}
+
+ant.fileScanner{
+ fileset(dir: baseDir){
+ extensions.each{
+ include(name: 'lucene/**/*.' + it)
+ include(name: 'solr/**/*.' + it)
+ include(name: 'dev-tools/**/*.' + it)
+ include(name: '*.' + it)
+ }
+ // TODO: For now we don't scan txt files, so we
+ // check licenses in top-level folders separately:
+ include(name: '*.txt')
+ include(name: '*/*.txt')
+ // excludes:
+ exclude(name: '**/build/**')
+ exclude(name: '**/dist/**')
+ exclude(name: 'lucene/benchmark/work/**')
+ exclude(name: 'lucene/benchmark/temp/**')
+ exclude(name: '**/CheckLoggingConfiguration.java')
+ exclude(name: 'lucene/tools/src/groovy/check-source-patterns.groovy') // ourselves :-)
+ }
+}.each{ f ->
+ task.log('Scanning file: ' + f, Project.MSG_VERBOSE);
+ def text = f.getText('UTF-8');
+ invalidPatterns.each{ pattern,name ->
+ if (pattern.matcher(text).find()) {
+ reportViolation(f, name);
+ }
+ }
+ def javadocsMatcher = javadocsPattern.matcher(text);
+ def ratDocument = new FileDocument(f);
+ while (javadocsMatcher.find()) {
+ if (isLicense(javadocsMatcher, ratDocument)) {
+ reportViolation(f, String.format(Locale.ENGLISH, 'javadoc-style license header [%s]',
+ ratDocument.getMetaData().value(MetaData.RAT_URL_LICENSE_FAMILY_NAME)));
+ }
+ }
+ if (f.name.endsWith('.java')) {
+ if (text.contains('org.slf4j.LoggerFactory')) {
+ if (!validLoggerPattern.matcher(text).find()) {
+ reportViolation(f, 'invalid logging pattern [not private static final, uses static class name]');
+ }
+ }
+ checkLicenseHeaderPrecedes(f, 'package', packagePattern, javaCommentPattern, text, ratDocument);
+ if (f.name.contains("Test")) {
+ checkMockitoAssume(f, text);
+ }
+ }
+ if (f.name.endsWith('.xml') || f.name.endsWith('.xml.template')) {
+ checkLicenseHeaderPrecedes(f, '<tag>', xmlTagPattern, xmlCommentPattern, text, ratDocument);
+ }
+ if (f.name.endsWith('.adoc')) {
+ checkForUnescapedSymbolSubstitutions(f, text);
+ }
+};
+
+if (found) {
+ throw new BuildException(String.format(Locale.ENGLISH, 'Found %d violations in source files (%s).',
+ found, violations.join(', ')));
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f2c4ca5e/lucene/tools/src/groovy/check-working-copy.groovy
----------------------------------------------------------------------
diff --git a/lucene/tools/src/groovy/check-working-copy.groovy b/lucene/tools/src/groovy/check-working-copy.groovy
new file mode 100644
index 0000000..079a18b
--- /dev/null
+++ b/lucene/tools/src/groovy/check-working-copy.groovy
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Task script that is called by Ant's build.xml file:
+ * Checks GIT working copy for unversioned or modified files.
+ */
+
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.Project;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.Status;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
+import org.eclipse.jgit.errors.*;
+
+def setProjectPropertyFromSet = { prop, set ->
+ if (set) {
+ properties[prop] = '* ' + set.join(properties['line.separator'] + '* ');
+ }
+};
+
+try {
+ task.log('Initializing working copy...', Project.MSG_INFO);
+ final Repository repository = new FileRepositoryBuilder()
+ .setWorkTree(project.getBaseDir())
+ .setMustExist(true)
+ .build();
+
+ task.log('Checking working copy status...', Project.MSG_INFO);
+ final Status status = new Git(repository).status().call();
+ if (!status.isClean()) {
+ final SortedSet unversioned = new TreeSet(), modified = new TreeSet();
+ status.properties.each{ prop, val ->
+ if (val instanceof Set) {
+ if (prop in ['untracked', 'untrackedFolders', 'missing']) {
+ unversioned.addAll(val);
+ } else if (prop != 'ignoredNotInIndex') {
+ modified.addAll(val);
+ }
+ }
+ };
+ setProjectPropertyFromSet('wc.unversioned.files', unversioned);
+ setProjectPropertyFromSet('wc.modified.files', modified);
+ }
+} catch (RepositoryNotFoundException | NoWorkTreeException | NotSupportedException e) {
+ task.log('WARNING: Development directory is not a valid GIT checkout! Disabling checks...', Project.MSG_WARN);
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f2c4ca5e/lucene/tools/src/groovy/install-markdown-filter.groovy
----------------------------------------------------------------------
diff --git a/lucene/tools/src/groovy/install-markdown-filter.groovy b/lucene/tools/src/groovy/install-markdown-filter.groovy
new file mode 100644
index 0000000..2b5544b
--- /dev/null
+++ b/lucene/tools/src/groovy/install-markdown-filter.groovy
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Task script that is called by Ant's common-build.xml file:
+ * Installs markdown filter into Ant.
+ */
+
+import org.apache.tools.ant.AntTypeDefinition;
+import org.apache.tools.ant.ComponentHelper;
+import org.apache.tools.ant.filters.TokenFilter.ChainableReaderFilter;
+import com.vladsch.flexmark.ast.Node;
+import com.vladsch.flexmark.ast.Heading;
+import com.vladsch.flexmark.html.HtmlRenderer;
+import com.vladsch.flexmark.parser.Parser;
+import com.vladsch.flexmark.parser.ParserEmulationProfile;
+import com.vladsch.flexmark.util.html.Escaping;
+import com.vladsch.flexmark.util.options.MutableDataSet;
+import com.vladsch.flexmark.ext.abbreviation.AbbreviationExtension;
+import com.vladsch.flexmark.ext.autolink.AutolinkExtension;
+
+public final class MarkdownFilter extends ChainableReaderFilter {
+ @Override
+ public String filter(String markdownSource) {
+ MutableDataSet options = new MutableDataSet();
+ options.setFrom(ParserEmulationProfile.MARKDOWN);
+ options.set(Parser.EXTENSIONS, [ AbbreviationExtension.create(), AutolinkExtension.create() ]);
+ options.set(HtmlRenderer.RENDER_HEADER_ID, true);
+ options.set(HtmlRenderer.MAX_TRAILING_BLANK_LINES, 0);
+ Node parsed = Parser.builder(options).build().parse(markdownSource);
+
+ StringBuilder html = new StringBuilder('<html>\n<head>\n');
+ CharSequence title = parsed.getFirstChildAny(Heading.class)?.getText();
+ if (title != null) {
+ html.append('<title>').append(Escaping.escapeHtml(title, false)).append('</title>\n');
+ }
+ html.append('<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">\n')
+ .append('</head>\n<body>\n');
+ HtmlRenderer.builder(options).build().render(parsed, html);
+ html.append('</body>\n</html>\n');
+ return html;
+ }
+}
+
+AntTypeDefinition t = new AntTypeDefinition();
+t.setName('markdownfilter');
+t.setClass(MarkdownFilter.class);
+ComponentHelper.getComponentHelper(project).addDataTypeDefinition(t);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f2c4ca5e/lucene/tools/src/groovy/patch-mrjar-classes.groovy
----------------------------------------------------------------------
diff --git a/lucene/tools/src/groovy/patch-mrjar-classes.groovy b/lucene/tools/src/groovy/patch-mrjar-classes.groovy
index d169997..07f285c 100644
--- a/lucene/tools/src/groovy/patch-mrjar-classes.groovy
+++ b/lucene/tools/src/groovy/patch-mrjar-classes.groovy
@@ -15,6 +15,11 @@
* limitations under the License.
*/
+/** Task script that is called by Ant's common-build.xml file:
+ * Patches Java 8 class files to replace method signatures by
+ * native Java 9 optimized ones (to be placed in MR-JAR).
+ */
+
import org.apache.tools.ant.Project;
import org.objectweb.asm.ClassReader;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f2c4ca5e/lucene/tools/src/groovy/run-beaster.groovy
----------------------------------------------------------------------
diff --git a/lucene/tools/src/groovy/run-beaster.groovy b/lucene/tools/src/groovy/run-beaster.groovy
new file mode 100644
index 0000000..f94a456
--- /dev/null
+++ b/lucene/tools/src/groovy/run-beaster.groovy
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Task script that is called by Ant's common-build.xml file:
+ * Runs test beaster.
+ */
+
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.BuildLogger;
+import org.apache.tools.ant.Project;
+
+int iters = (properties['beast.iters'] ?: '1') as int;
+if (iters <= 1) {
+ throw new BuildException("Please give -Dbeast.iters with an int value > 1.");
+}
+
+def antcall = project.createTask('antcall');
+antcall.with {
+ target = '-test';
+ inheritAll = true;
+ inheritRefs = true;
+ createParam().with {
+ name = "tests.isbeasting";
+ value = "true";
+ };
+};
+
+(1..iters).each { i ->
+ task.log('Beast round: ' + i, Project.MSG_INFO);
+ try {
+ // disable verbose build logging:
+ project.buildListeners.each { listener ->
+ if (listener instanceof BuildLogger) {
+ listener.messageOutputLevel = Project.MSG_WARN;
+ }
+ };
+
+ antcall.execute();
+
+ } catch (BuildException be) {
+ def logFile = new File(properties["junit.output.dir"], "tests-failures.txt");
+ if (logFile.exists()) {
+ logFile.eachLine("UTF-8", { line ->
+ task.log(line, Project.MSG_ERR);
+ });
+ }
+ throw be;
+ } finally {
+ // restore build logging (unfortunately there is no way to get the original logging level (write-only property):
+ project.buildListeners.each { listener ->
+ if (listener instanceof BuildLogger) {
+ listener.messageOutputLevel = Project.MSG_INFO;
+ }
+ };
+ }
+};
+task.log('Beasting finished.', Project.MSG_INFO);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f2c4ca5e/lucene/tools/src/groovy/run-maven-build.groovy
----------------------------------------------------------------------
diff --git a/lucene/tools/src/groovy/run-maven-build.groovy b/lucene/tools/src/groovy/run-maven-build.groovy
new file mode 100644
index 0000000..c26c7bf
--- /dev/null
+++ b/lucene/tools/src/groovy/run-maven-build.groovy
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Task script that is called by Ant's build.xml file:
+ * Runs maven build from within Ant after creating POMs.
+ */
+
+import groovy.xml.NamespaceBuilder;
+import org.apache.tools.ant.Project;
+
+def userHome = properties['user.home'], commonDir = properties['common.dir'];
+def propPrefix = '-mvn.inject.'; int propPrefixLen = propPrefix.length();
+
+def subProject = project.createSubProject();
+project.copyUserProperties(subProject);
+subProject.initProperties();
+new AntBuilder(subProject).sequential{
+ property(file: userHome+'/lucene.build.properties', prefix: propPrefix);
+ property(file: userHome+'/build.properties', prefix: propPrefix);
+ property(file: commonDir+'/build.properties', prefix: propPrefix);
+};
+
+def cmdlineProps = subProject.properties
+ .findAll{ k, v -> k.startsWith(propPrefix) }
+ .collectEntries{ k, v -> [k.substring(propPrefixLen), v] };
+cmdlineProps << project.userProperties.findAll{ k, v -> !k.startsWith('ant.') };
+
+def artifact = NamespaceBuilder.newInstance(ant, 'antlib:org.apache.maven.artifact.ant');
+
+task.log('Running Maven with props: ' + cmdlineProps.toString(), Project.MSG_INFO);
+artifact.mvn(pom: properties['maven-build-dir']+'/pom.xml', mavenVersion: properties['maven-version'], failonerror: true, fork: true) {
+ cmdlineProps.each{ k, v -> arg(value: '-D' + k + '=' + v) };
+ arg(value: '-fae');
+ arg(value: 'install');
+};