You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by cw...@apache.org on 2021/05/21 08:28:41 UTC
[druid] branch chill-travis created (now 28e8012)
This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a change to branch chill-travis
in repository https://gitbox.apache.org/repos/asf/druid.git.
at 28e8012 only skip stuff for PR builds, add some tests
This branch includes the following new commits:
new 28e8012 only skip stuff for PR builds, add some tests
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
[druid] 01/01: only skip stuff for PR builds, add some tests
Posted by cw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch chill-travis
in repository https://gitbox.apache.org/repos/asf/druid.git
commit 28e801200e1a1bfe3ba2cd3b79690c11a006bbb8
Author: Clint Wylie <cw...@apache.org>
AuthorDate: Fri May 21 01:27:51 2021 -0700
only skip stuff for PR builds, add some tests
---
.idea/inspectionProfiles/Druid.xml | 6 +--
.travis.yml | 18 ++++----
check-test-suite.py => check_test_suite.py | 50 +++++++++++++---------
check_test_suite_test.py | 67 ++++++++++++++++++++++++++++++
4 files changed, 111 insertions(+), 30 deletions(-)
diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml
index 2da1cb7..bc1782f 100644
--- a/.idea/inspectionProfiles/Druid.xml
+++ b/.idea/inspectionProfiles/Druid.xml
@@ -59,6 +59,7 @@
<inspection_tool class="EqualsUsesNonFinalVariable" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="EqualsWhichDoesntCheckParameterClass" enabled="true" level="ERROR" enabled_by_default="true" />
<inspection_tool class="EqualsWithItself" enabled="true" level="ERROR" enabled_by_default="true" />
+ <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="FieldAccessNotGuarded" enabled="true" level="ERROR" enabled_by_default="true" />
<inspection_tool class="FieldCanBeLocal" enabled="true" level="WARNING" enabled_by_default="true">
<option name="EXCLUDE_ANNOS">
@@ -70,7 +71,6 @@
</option>
<option name="IGNORE_FIELDS_USED_IN_MULTIPLE_METHODS" value="true" />
</inspection_tool>
- <inspection_tool class="FieldMayBeFinal" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="FinalStaticMethod" enabled="true" level="ERROR" enabled_by_default="true" />
<inspection_tool class="FlowJSError" enabled="false" level="Non-TeamCity Error" enabled_by_default="false" />
<inspection_tool class="ForCanBeForeach" enabled="true" level="WARNING" enabled_by_default="true">
@@ -470,8 +470,8 @@
<inspection_tool class="XmlHighlighting" enabled="true" level="Non-TeamCity Warning" enabled_by_default="true" />
<inspection_tool class="XmlInvalidId" enabled="true" level="Non-TeamCity Error" enabled_by_default="true" />
<inspection_tool class="XmlPathReference" enabled="true" level="Non-TeamCity Error" enabled_by_default="true" />
- <inspection_tool class="unused" enabled="true" level="WARNING" enabled_by_default="true" isSelected="false">
- <scope name="UnusedInspectionsScope" level="ERROR" enabled="true" isSelected="false">
+ <inspection_tool class="unused" enabled="true" level="WARNING" enabled_by_default="true">
+ <scope name="UnusedInspectionsScope" level="ERROR" enabled="true">
<option name="LOCAL_VARIABLE" value="true" />
<option name="FIELD" value="true" />
<option name="METHOD" value="true" />
diff --git a/.travis.yml b/.travis.yml
index 2792aaf..5178095 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,7 +42,7 @@ addons:
# Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not
# needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any
# output. To compensate, use travis_wait to extend the timeout.
-install: ./check-test-suite.py && travis_terminate 0 || echo 'Running maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+install: ./check_test_suite.py && travis_terminate 0 || echo 'Running maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
# There are 3 stages of tests
# 1. Tests - phase 1
@@ -88,7 +88,7 @@ jobs:
install: skip
before_script: &setup_generate_license
- sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y
- - ./check-test-suite.py && travis_terminate 0 || echo 'Continuing setup'
+ - ./check_test_suite.py && travis_terminate 0 || echo 'Continuing setup'
- pip3 install wheel # install wheel first explicitly
- pip3 install pyyaml
script:
@@ -102,11 +102,15 @@ jobs:
- distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2
- distribution/bin/check-licenses.py licenses.yaml target/license-reports
+ - name: "script checks"
+ install: skip
+ script: ./check_test_suite_test.py
+
- name: "(openjdk8) strict compilation"
install: skip
# Strict compilation requires more than 2 GB
script: >
- ./check-test-suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -Pstrict compile test-compile --fail-at-end
+ ./check_test_suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -Pstrict compile test-compile --fail-at-end
-pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
- name: "analyze dependencies"
@@ -133,7 +137,7 @@ jobs:
- name: "intellij inspections"
script: >
- ./check-test-suite.py && travis_terminate 0 || docker run --rm
+ ./check_test_suite.py && travis_terminate 0 || docker run --rm
-v $(pwd):/project
-v ~/.m2:/home/inspect/.m2
ccaominh/intellij-inspect:1.0.0
@@ -301,7 +305,7 @@ jobs:
install: skip
stage: Tests - phase 1
script:
- - ./check-test-suite.py && travis_terminate 0 || ${MVN} test -pl 'web-console'
+ - ./check_test_suite.py && travis_terminate 0 || ${MVN} test -pl 'web-console'
after_success:
- (cd web-console && travis_retry npm run codecov) # retry in case of network error
@@ -309,14 +313,14 @@ jobs:
before_install: *setup_generate_license
install: web-console/script/druid build
before_script:
- - ./check-test-suite.py && travis_terminate 0 || echo 'Starting nvm install...'
+ - ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...'
- nvm install 10.24.0
- web-console/script/druid start
script: (cd web-console && npm run test-e2e)
after_script: web-console/script/druid stop
- name: "docs"
- install: ./check-test-suite.py && travis_terminate 0 || (cd website && npm install)
+ install: ./check_test_suite.py && travis_terminate 0 || (cd website && npm install)
script: |-
(cd website && npm run lint && npm run spellcheck) || { echo "
diff --git a/check-test-suite.py b/check_test_suite.py
similarity index 71%
rename from check-test-suite.py
rename to check_test_suite.py
index 5006e92..b067c27 100755
--- a/check-test-suite.py
+++ b/check_test_suite.py
@@ -27,7 +27,7 @@ always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11)
# of CI can be skipped
ignore_prefixes = ['.github', '.idea', '.asf.yaml', '.backportrc.json', '.codecov.yml', '.dockerignore', '.gitignore',
'.lgtm.yml', 'CONTRIBUTING.md', 'setup-hooks.sh', 'upload.sh', 'dev', 'distribution/docker',
- 'distribution/asf-release-process-guide.md', '.travis.yml', 'check-test-suite.py']
+ 'distribution/asf-release-process-guide.md', '.travis.yml', 'check_test_suite.py']
# these files are docs changes
# if changes are limited to this set then we can skip web-console and java
# if no changes in this set we can skip docs
@@ -60,11 +60,15 @@ def check_console(file):
print("found web-console file change: {}".format(file))
return is_console
+
+# given a test suite name (travis job name) and set of files in a git diff
+# try to determine if the test suite should run or not
def check_should_run_suite(suite, diff_files):
if suite in always_run_jobs:
# you gotta do what you gotta do
return True
+ # is everything in the ignorable list
all_ignore = True
any_docs = False
all_docs = True
@@ -94,32 +98,38 @@ def check_should_run_suite(suite, diff_files):
return True
def failWithUsage():
- sys.stderr.write("usage: check-test-suite.py <test-suite-name>\n")
- sys.stderr.write(" e.g., check-test-suite.py docs")
+ sys.stderr.write("usage: check_test_suite.py <test-suite-name>\n")
+ sys.stderr.write(" e.g., check_test_suite.py docs")
sys.exit(1)
-suite_name = ""
+if __name__ == '__main__':
+ suite_name = ""
-if len(sys.argv) == 1:
- if 'TRAVIS_JOB_NAME' in os.environ:
- suite_name = os.environ['TRAVIS_JOB_NAME']
+ if len(sys.argv) == 1:
+ if 'TRAVIS_JOB_NAME' in os.environ:
+ suite_name = os.environ['TRAVIS_JOB_NAME']
+ else:
+ failWithUsage()
+ elif len(sys.argv) == 2:
+ suite_name = sys.argv[1]
else:
failWithUsage()
-elif len(sys.argv) == 2:
- suite_name = sys.argv[1]
-else:
- failWithUsage()
-all_changed_files_string = subprocess.check_output("git diff --name-only HEAD~1", shell=True).decode('UTF-8')
-all_changed_files = all_changed_files_string.splitlines()
-print("Checking if suite '{}' needs to run test on diff:\n{}".format(suite_name, all_changed_files_string))
-needs_run = check_should_run_suite(suite_name, all_changed_files)
+ all_changed_files_string = subprocess.check_output("git diff --name-only HEAD~1", shell=True).decode('UTF-8')
+ all_changed_files = all_changed_files_string.splitlines()
+ print("Checking if suite '{}' needs to run test on diff:\n{}".format(suite_name, all_changed_files_string))
-if needs_run:
- print("Changes detected, need to run test suite '{}'".format(suite_name))
- sys.exit(1)
+ # we should always run all test suites for builds that are not for a pull request
+ is_pr = False
+ if 'TRAVIS_PULL_REQUEST' in os.environ and os.environ['TRAVIS_PULL_REQUEST'] != 'false':
+ is_pr = True
+ needs_run = not is_pr or check_should_run_suite(suite_name, all_changed_files)
+
+ if needs_run:
+ print("Changes detected, need to run test suite '{}'".format(suite_name))
+ sys.exit(1)
-print("No applicable changes detected, can skip test suite '{}'".format(suite_name))
-sys.exit(0)
+ print("No applicable changes detected, can skip test suite '{}'".format(suite_name))
+ sys.exit(0)
diff --git a/check_test_suite_test.py b/check_test_suite_test.py
new file mode 100755
index 0000000..6c2e4f6
--- /dev/null
+++ b/check_test_suite_test.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import check_test_suite
+
+class CheckTestSuite(unittest.TestCase):
+ def test_docs(self):
+ self.assertEqual(False, check_test_suite.check_docs('.travis.yml'))
+ self.assertEqual(False, check_test_suite.check_docs('check_test_suite_test.py'))
+ self.assertEqual(True, check_test_suite.check_docs('website/core/Footer.js'))
+ self.assertEqual(True, check_test_suite.check_docs('docs/ingestion/index.md'))
+
+ for job in check_test_suite.docs_jobs:
+ self.assertEqual(
+ True,
+ check_test_suite.check_should_run_suite(job, ['check_test_suite_test.py', 'docs/ingestion/index.md'])
+ )
+ self.assertEqual(
+ False,
+ check_test_suite.check_should_run_suite(
+ job,
+ ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java']
+ )
+ )
+
+ def test_web_console(self):
+ self.assertEqual(False, check_test_suite.check_console('.travis.yml'))
+ self.assertEqual(False, check_test_suite.check_console('check_test_suite_test.py'))
+ self.assertEqual(False, check_test_suite.check_console('website/core/Footer.js'))
+ self.assertEqual(True, check_test_suite.check_console('web-console/assets/azure.png'))
+ self.assertEqual(True, check_test_suite.check_console('web-console/src/views/index.ts'))
+ self.assertEqual(True, check_test_suite.check_console('web-console/unified-console.html'))
+
+ for job in check_test_suite.web_console_jobs:
+ self.assertEqual(
+ True,
+ check_test_suite.check_should_run_suite(
+ job,
+ ['check_test_suite_test.py', 'web-console/unified-console.html']
+ )
+ )
+ self.assertEqual(
+ False,
+ check_test_suite.check_should_run_suite(
+ job,
+ ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java']
+ )
+ )
+
+
+if __name__ == '__main__':
+ unittest.main()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org