You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by ca...@apache.org on 2022/10/11 07:41:12 UTC
[incubator-linkis-website] branch dev updated: Add an crawler action to workflow (#514)
This is an automated email from the ASF dual-hosted git repository.
casion pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-linkis-website.git
The following commit(s) were added to refs/heads/dev by this push:
new fc7bdb8ea5 Add an crawler action to workflow (#514)
fc7bdb8ea5 is described below
commit fc7bdb8ea5cda0b56f2991b90cab1e817463b8fd
Author: Beacontownfc <89...@users.noreply.github.com>
AuthorDate: Tue Oct 11 15:41:06 2022 +0800
Add an crawler action to workflow (#514)
* add crawler
* Update scheduled-crawl.yml
---
.github/actions/Contributor_crawler/Dockerfile | 4 +++
.github/actions/Contributor_crawler/action.yml | 16 ++++++++++++
.github/actions/Contributor_crawler/main.js | 26 +++++++++++++++++++
.github/workflows/scheduled-crawl.yml | 36 ++++++++++++++++++++++++++
4 files changed, 82 insertions(+)
diff --git a/.github/actions/Contributor_crawler/Dockerfile b/.github/actions/Contributor_crawler/Dockerfile
new file mode 100644
index 0000000000..63243c09c1
--- /dev/null
+++ b/.github/actions/Contributor_crawler/Dockerfile
@@ -0,0 +1,4 @@
+FROM node:16.15.0
+COPY . .
+
+CMD [ "node", "/main.js"]
\ No newline at end of file
diff --git a/.github/actions/Contributor_crawler/action.yml b/.github/actions/Contributor_crawler/action.yml
new file mode 100644
index 0000000000..37c25469cc
--- /dev/null
+++ b/.github/actions/Contributor_crawler/action.yml
@@ -0,0 +1,16 @@
+name: "Github Crawler"
+
+description: "Auto crawl github repo contributors."
+
+inputs:
+ repo:
+ description: "The repo that needs to crawl."
+ required: true
+ num:
+ description: "How many contributors need to be crawled."
+ required: true
+
+
+runs:
+ using: "docker"
+ image: "Dockerfile"
\ No newline at end of file
diff --git a/.github/actions/Contributor_crawler/main.js b/.github/actions/Contributor_crawler/main.js
new file mode 100644
index 0000000000..75c3df279d
--- /dev/null
+++ b/.github/actions/Contributor_crawler/main.js
@@ -0,0 +1,26 @@
+const https = require('https');
+const fs = require('fs');
+let num = process.env['INPUT_NUM'];
+let repo = process.env['INPUT_REPO'];
+
+let name = repo == 'apache/incubator-linkis-website' ? 'contributorsWebsite.json' : 'contributors.json'
+const options = {headers: {'User-Agent': 'request'}};
+
+// repo such as apache/incubator-linkis-website
+https.get(`https://api.github.com/repos/${repo}/contributors?page=1&per_page=${num}`, options, res => {
+ let chunks = [];
+ res.on('data', chunk => {
+ chunks.push(chunk);
+ });
+ res.on('end', () => {
+ let jsonstr = Buffer.concat(chunks).toString('utf-8');
+ fs.writeFileSync(`./src/components/${name}`, jsonstr, (err) => {
+ if(err) console.log(err);
+ })
+ })
+}).on('error', err => {
+ console.log('Error: ', err.message);
+});
+
+
+
diff --git a/.github/workflows/scheduled-crawl.yml b/.github/workflows/scheduled-crawl.yml
new file mode 100644
index 0000000000..43a91c3cf0
--- /dev/null
+++ b/.github/workflows/scheduled-crawl.yml
@@ -0,0 +1,36 @@
+name: Scheduled crawl
+
+on:
+ schedule:
+ # Run everyday at 0:00 AM or 12:00 AM (See https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07)
+ - cron: "0 0,12 * * *"
+
+jobs:
+ # The purpose of this workflow is to periodically merge "asf-staging" into "asf-site"
+ build:
+ # The type of runner that the job will run on
+ runs-on: ubuntu-latest
+
+ # Steps represent a sequence of tasks that will be executed as part of the job
+ steps:
+ # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+ - uses: actions/checkout@v3
+ with:
+ ref: 'dev'
+
+ - name: scheduled crawl linkis website contributor
+ uses: ./.github/actions/Contributor_crawler
+ with:
+ repo: apache/incubator-linkis-website
+ num: 1000
+
+ - name: scheduled crawl linkis contributor
+ uses: ./.github/actions/Contributor_crawler
+ with:
+ repo: apache/incubator-linkis
+ num: 1000
+
+ - name: scheduled commit contributor
+ uses: EndBug/add-and-commit@v9
+ with:
+ default_author: github_actions
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org