You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by la...@apache.org on 2021/02/23 20:56:44 UTC
[incubator-mxnet] 02/02: Remove unmaintained BLC (#19801)
This is an automated email from the ASF dual-hosted git repository.
lausen pushed a commit to branch v1.x
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
commit e1d110531915470c31c1af7eed875fb4cd592814
Author: Leonard Lausen <la...@amazon.com>
AuthorDate: Thu Jan 28 19:17:09 2021 +0100
Remove unmaintained BLC (#19801)
qrgfqwrgq
---
ci/docker/Dockerfile.build.ubuntu_blc | 41 --------
ci/docker/install/ubuntu_npm_blc.sh | 37 --------
ci/docker/runtime_functions.sh | 13 +--
.../broken_link_checker_test/JenkinsfileForBLC | 59 ------------
tests/nightly/broken_link_checker_test/README.md | 27 ------
.../broken_link_checker.sh | 30 ------
.../broken_link_checker_test/test_broken_links.py | 105 ---------------------
7 files changed, 1 insertion(+), 311 deletions(-)
diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/docker/Dockerfile.build.ubuntu_blc
deleted file mode 100644
index a413648..0000000
--- a/ci/docker/Dockerfile.build.ubuntu_blc
+++ /dev/null
@@ -1,41 +0,0 @@
-# -*- mode: dockerfile -*-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
-
-FROM ubuntu:16.04
-
-WORKDIR /work/deps
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-COPY install/ubuntu_python.sh /work/
-COPY install/requirements /work/
-RUN /work/ubuntu_python.sh
-COPY install/ubuntu_npm_blc.sh /work/
-RUN /work/ubuntu_npm_blc.sh
-
-ARG USER_ID=0
-ARG GROUP_ID=0
-COPY install/ubuntu_adduser.sh /work/
-RUN /work/ubuntu_adduser.sh
-
-COPY runtime_functions.sh /work/
-
-WORKDIR /work/mxnet
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/ubuntu_npm_blc.sh b/ci/docker/install/ubuntu_npm_blc.sh
deleted file mode 100755
index 59caa9f..0000000
--- a/ci/docker/install/ubuntu_npm_blc.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-echo 'Installing npm...'
-apt-get update || true
-apt-get install -y npm
-
-echo "Obtaining NodeJS version 8.x"
-curl -sL https://deb.nodesource.com/setup_8.x | bash -
-
-echo "Installing nodejs"
-apt-get install -y nodejs
-
-# Install broken link checker utility
-echo "Installing broken link checker utility"
-npm install broken-link-checker -g
-
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index b3b040b..2f7ad65 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -947,10 +947,6 @@ build_ubuntu_gpu_large_tensor() {
ninja
}
-build_ubuntu_blc() {
- echo "pass"
-}
-
# Testing
sanity_check() {
@@ -2102,14 +2098,7 @@ publish_scala_deploy() {
popd
}
-# broken_link_checker
-
-broken_link_checker() {
- set -ex
- ./tests/nightly/broken_link_checker_test/broken_link_checker.sh
-}
-
-# artifact repository unit tets
+# artifact repository unit tests
test_artifact_repository() {
set -ex
pushd .
diff --git a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC b/tests/nightly/broken_link_checker_test/JenkinsfileForBLC
deleted file mode 100755
index 8a96a4f..0000000
--- a/tests/nightly/broken_link_checker_test/JenkinsfileForBLC
+++ /dev/null
@@ -1,59 +0,0 @@
-// -*- mode: groovy -*-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-//This is a Jenkinsfile for the broken link checker test.
-
-node('utility') {
- // Loading the utilities requires a node context unfortunately
- checkout scm
- utils = load('ci/Jenkinsfile_utils.groovy')
-}
-utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu')
-
-utils.main_wrapper(
-core_logic: {
- stage('BLC'){
- parallel 'BrokenLinkChecker: CPU': {
- node(NODE_LINUX_CPU) {
- ws('workspace/brokenLinkChecker') {
- timeout(time: 60, unit: 'MINUTES') {
- try {
- utils.init_git()
- utils.docker_run('ubuntu_blc', 'broken_link_checker', false)
- } catch(Exception ex) {
- sh "echo Broken Link Checker failed with exception - " + ex.toString()
- }
- }
- }
- }
- }
- }
-}
-,
-failure_handler:
-{
- if (currentBuild.result == "FAILURE") {
- emailext body: '''https://mxnet.apache.org broken link test summary:
- |
- |Please view the logs at ${BUILD_URL}
- |
- |${BUILD_LOG_EXCERPT, start="START - Broken links summary", end="END - Broken links summary"}''',
- replyTo: '${EMAIL}', subject: '[BROKEN LINK CHECKER FAILED] Run ${BUILD_NUMBER}', to: '${EMAIL}'
- }
-}
-)
diff --git a/tests/nightly/broken_link_checker_test/README.md b/tests/nightly/broken_link_checker_test/README.md
deleted file mode 100644
index d493390..0000000
--- a/tests/nightly/broken_link_checker_test/README.md
+++ /dev/null
@@ -1,27 +0,0 @@
-<!--- Licensed to the Apache Software Foundation (ASF) under one -->
-<!--- or more contributor license agreements. See the NOTICE file -->
-<!--- distributed with this work for additional information -->
-<!--- regarding copyright ownership. The ASF licenses this file -->
-<!--- to you under the Apache License, Version 2.0 (the -->
-<!--- "License"); you may not use this file except in compliance -->
-<!--- with the License. You may obtain a copy of the License at -->
-
-<!--- http://www.apache.org/licenses/LICENSE-2.0 -->
-
-<!--- Unless required by applicable law or agreed to in writing, -->
-<!--- software distributed under the License is distributed on an -->
-<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
-<!--- KIND, either express or implied. See the License for the -->
-<!--- specific language governing permissions and limitations -->
-<!--- under the License. -->
-
-# Broken link checker test
-
-This folder contains the scripts that are required to run the nightly job of checking the broken links.
-
-## JenkinsfileForBLC
-This is configuration file for jenkins job.
-
-## Details
-The `broken_link_checker.sh` is a top level script that invokes the `test_broken_links.py` and `check_regression.sh` scripts.
-The `test_broken_links.py` invokes broken link checker tool (blc) from nodeJs and reports the list of URLs that are not accessible.
diff --git a/tests/nightly/broken_link_checker_test/broken_link_checker.sh b/tests/nightly/broken_link_checker_test/broken_link_checker.sh
deleted file mode 100755
index 450cd65..0000000
--- a/tests/nightly/broken_link_checker_test/broken_link_checker.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#Author: Amol Lele
-
-#software-properties-common, curl, npm are installed in the docker container 'ubuntu_blc'
-
-echo "Invoking broken_link_checker.sh script"
-echo `pwd`
-cd tests/nightly/broken_link_checker_test
-echo `pwd`
-
-echo "Running test_broken_links.py"
-python test_broken_links.py
diff --git a/tests/nightly/broken_link_checker_test/test_broken_links.py b/tests/nightly/broken_link_checker_test/test_broken_links.py
deleted file mode 100755
index 2c0634f..0000000
--- a/tests/nightly/broken_link_checker_test/test_broken_links.py
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-import sys
-from subprocess import check_output, STDOUT, CalledProcessError
-
-
-def prepare_link_test_result(command_output):
- # Constant string patterns
- NEW_PAGE_TEST_START_REGEX = "Getting links from:"
- BROKEN_PAGE_START_REGEX = "BROKEN"
- PAGE_TEST_END_REGEX = "Finished! "
-
- # Whitelisted broken links patterns
- HTTP_403_REGEX = "(HTTP_403)"
- HTTP_401_REGEX = "(HTTP_401)"
- HTTP_409_REGEX = "(HTTP_409)"
- HTTP_3XX_REGEX = "(HTTP_3"
- BLC_UNKNOWN_REGEX = "(BLC_UNKNOWN)"
- HTTP_UNDEFINED = "HTTP_undefined"
- FALSE_SCALA_API_DOC_LINK = "java$lang.html"
- FALSE_SCALA_API_DEPRECATED_LINK = "api/scala/docs/deprecated-list.html"
- FALSE_PAPER_LINK = "https://static.googleusercontent.com/media/research.google.com/en/pubs/archive/45488.pdf"
-
- # Initialize flags with happy case
- current_page = ""
- current_page_broken = False
- current_page_broken_links = ""
-
- broken_links_count = 0
- broken_links_summary = ""
-
- for line in command_output.splitlines():
- if line.startswith(NEW_PAGE_TEST_START_REGEX):
- # New page test is starting. Reset the flags
- current_page = line.split(NEW_PAGE_TEST_START_REGEX)[1]
- current_page_broken = False
- current_page_broken_links = ""
-
- if line.find(BROKEN_PAGE_START_REGEX) != -1:
- # Skip (401, 403, 409, unknown issues)
- if HTTP_403_REGEX not in line and HTTP_401_REGEX not in line and HTTP_409_REGEX not in line and HTTP_3XX_REGEX not in line and BLC_UNKNOWN_REGEX not in line and HTTP_UNDEFINED not in line and FALSE_SCALA_API_DOC_LINK not in line and FALSE_SCALA_API_DEPRECATED_LINK not in line and FALSE_PAPER_LINK not in line:
- current_page_broken = True
- current_page_broken_links += line.split(BROKEN_PAGE_START_REGEX)[1] + "\n"
-
- if line.startswith(PAGE_TEST_END_REGEX):
- if current_page_broken:
- broken_links_count += 1
- broken_links_summary += "\nURL - " + current_page
- broken_links_summary += "\nBroken Links\n" + current_page_broken_links
-
- return broken_links_count, broken_links_summary
-
-# Command to check broken links
-# Reference - https://www.npmjs.com/package/broken-link-checker
-cmd = "blc https://mxnet.apache.org -ro"
-broken_links_count = 0
-broken_links_summary = ""
-
-text_file = open("./blc_output.txt", 'w')
-command_output = ""
-print("Starting broken link test with command $ " + cmd)
-try:
- command_output = check_output(cmd, stderr=STDOUT, shell=True)
-except CalledProcessError as ex:
- if ex.returncode > 1:
- print("Failed to do broken link test. Console output : \n" + ex.output)
- sys.exit(ex.returncode)
- command_output = ex.output
-
-text_file.write(command_output)
-text_file.close()
-broken_links_count, broken_links_summary = prepare_link_test_result(command_output)
-# These START and END string in output is used to parse the script output in automated scripts and nightly jobs.
-print("START - Broken links count")
-print(broken_links_count)
-print("END - Broken links count")
-
-print("START - Broken links summary")
-
-if broken_links_count == 0:
- print("No broken links in https://mxnet.apache.org")
- print("END - Broken links summary")
-else:
- print(broken_links_summary)
- print("END - Broken links summary")
- # Fail the job as we found the broken links
- sys.exit(-1)