You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by gi...@apache.org on 2020/01/10 09:57:20 UTC
[incubator-dolphinscheduler-website] branch asf-site updated:
Automated deployment: Fri Jan 10 09:57:07 UTC 2020
0ab48e04bb13f127d28e241dbabf52e569cc6014
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler-website.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 20a2e36 Automated deployment: Fri Jan 10 09:57:07 UTC 2020 0ab48e04bb13f127d28e241dbabf52e569cc6014
20a2e36 is described below
commit 20a2e36449d041e036b17a8b662306eb0d6edad2
Author: dailidong <da...@users.noreply.github.com>
AuthorDate: Fri Jan 10 09:57:07 2020 +0000
Automated deployment: Fri Jan 10 09:57:07 UTC 2020 0ab48e04bb13f127d28e241dbabf52e569cc6014
---
build/blog.js | 2 +-
build/blogDetail.js | 2 +-
build/community.js | 2 +-
build/documentation.js | 2 +-
build/home.js | 2 +-
.../developer_guide/dolphin-scheduler-setup.html | 227 +++++++++++++++++++++
.../developer_guide/dolphin-scheduler-setup.json | 6 +
7 files changed, 238 insertions(+), 5 deletions(-)
diff --git a/build/blog.js b/build/blog.js
index 055871c..7a3b803 100644
--- a/build/blog.js
+++ b/build/blog.js
@@ -8,7 +8,7 @@ object-assign
(c) Sindre Sorhus
@license MIT
*/
-var o=Object.getOwnPropertySymbols,a=Object.prototype.hasOwnProperty,i=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
+var o=Object.getOwnPropertySymbols,a=Object.prototype.hasOwnProperty,i=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
* react-is.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
diff --git a/build/blogDetail.js b/build/blogDetail.js
index 6c21ace..579167d 100644
--- a/build/blogDetail.js
+++ b/build/blogDetail.js
@@ -8,7 +8,7 @@ object-assign
(c) Sindre Sorhus
@license MIT
*/
-var o=Object.getOwnPropertySymbols,i=Object.prototype.hasOwnProperty,a=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
+var o=Object.getOwnPropertySymbols,i=Object.prototype.hasOwnProperty,a=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
* react-is.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
diff --git a/build/community.js b/build/community.js
index e94033d..6051dc0 100644
--- a/build/community.js
+++ b/build/community.js
@@ -8,7 +8,7 @@ object-assign
(c) Sindre Sorhus
@license MIT
*/
-var o=Object.getOwnPropertySymbols,i=Object.prototype.hasOwnProperty,a=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
+var o=Object.getOwnPropertySymbols,i=Object.prototype.hasOwnProperty,a=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
* react-is.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
diff --git a/build/documentation.js b/build/documentation.js
index 103c27e..57877f3 100644
--- a/build/documentation.js
+++ b/build/documentation.js
@@ -8,7 +8,7 @@ object-assign
(c) Sindre Sorhus
@license MIT
*/
-var o=Object.getOwnPropertySymbols,i=Object.prototype.hasOwnProperty,a=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
+var o=Object.getOwnPropertySymbols,i=Object.prototype.hasOwnProperty,a=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
* react-is.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
diff --git a/build/home.js b/build/home.js
index 58a894c..23e8731 100644
--- a/build/home.js
+++ b/build/home.js
@@ -8,7 +8,7 @@ object-assign
(c) Sindre Sorhus
@license MIT
*/
-var o=Object.getOwnPropertySymbols,a=Object.prototype.hasOwnProperty,i=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
+var o=Object.getOwnPropertySymbols,a=Object.prototype.hasOwnProperty,i=Object.prototype.propertyIsEnumerable;e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map(function(e){return t[e]}).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach(function(e){r[e]=e}),"abcdefghijklmn [...]
* react-is.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
diff --git a/zh-cn/docs/developer_guide/dolphin-scheduler-setup.html b/zh-cn/docs/developer_guide/dolphin-scheduler-setup.html
new file mode 100644
index 0000000..f4823f5
--- /dev/null
+++ b/zh-cn/docs/developer_guide/dolphin-scheduler-setup.html
@@ -0,0 +1,227 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
+ <meta name="keywords" content="dolphin-scheduler-setup" />
+ <meta name="description" content="dolphin-scheduler-setup" />
+ <!-- 网页标签标题 -->
+ <title>dolphin-scheduler-setup</title>
+ <link rel="shortcut icon" href="/img/docsite.ico"/>
+ <link rel="stylesheet" href="/build/documentation.css" />
+</head>
+<body>
+ <div id="root"><div class="documentation-page" data-reactroot=""><header class="header-container header-container-normal"><div class="header-body"><a href="/zh-cn/index.html"><img class="logo" src="/img/hlogo_colorful.svg"/></a><div class="search search-normal"><span class="icon-search"></span></div><span class="language-switch language-switch-normal">En</span><div class="header-menu"><img class="header-menu-toggle" src="/img/system/menu_gray.png"/><div><ul class="ant-menu blackClass an [...]
+<ol>
+<li>fork <a href="https://github.com/apache/incubator-dolphinscheduler">dolphinscheduler</a></li>
+<li>git clone xxx，</li>
+<li>安装好MySQL,Zookeeper,Nginx，Hadoop(可选)。</li>
+<li>git clone项目后，进入目录，执行以下命令。</li>
+</ol>
+<pre><code>1. git branch -a #查看分支
+2. git checkout dev-db #切换到dev-db分支
+3. git pull #同步分支
+4. mvn -U clean package assembly:assembly -Dmaven.test.skip=true #编译项目。由于项目前端的日志模块，使用了gRPC调用后端，所以需要先编译项目。
+</code></pre>
+<h4>搭建后端</h4>
+<ol>
+<li>
+<p>创建ds数据库CREATE DATABASE dolphinscheduler。</p>
+</li>
+<li>
+<p>创建表和初始化数据：
+修改dao模块resource目录下application.properties文件中的数据库配置信息，然后执行org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler的类，运行完，刷新数据库，表和数据都有了。</p>
+</li>
+<li>
+<p>启动MasterServer
+org.apache.dolphinscheduler.server.master.MasterServer类main函数增加如下代码：</p>
+<pre><code>System.setProperty("spring.profiles.active","master");
+</code></pre>
+<p>修改server模块resources目录下master_logback.xml文件，增加以下代码：</p>
+<pre><code><root level="INFO">
+ <appender-ref ref="MASTERLOGFILE"/>
+ <!-- 增加日志到控制台-->
+ <appender-ref ref="STDOUT"/>
+</root>
+</code></pre>
+<p>修改common模块，quartz.properties中数据库配置信息，zookeeper.properties中链接信息(zookeeper.quorum), 然后执行MasterServer即可。</p>
+</li>
+<li>
+<p>启动WorkerServer
+org.apache.dolphinscheduler.server.worker.WorkerServer类main函数增加如下代码：</p>
+<pre><code>System.setProperty("spring.profiles.active","worker");
+</code></pre>
+<p>修改server模块resources目录下worker_logback.xml文件，增加以下代码：</p>
+<pre><code><root level="INFO">
+ <appender-ref ref="TASKLOGFILE"/>
+ <appender-ref ref="WORKERLOGFILE"/>
+ <!-- 增加日志到控制台-->
+ <appender-ref ref="STDOUT"/>
+</root>
+</code></pre>
+<p>然后执行MasterServer即可。</p>
+</li>
+</ol>
+<h4>搭建前端</h4>
+<p>进入dolphinscheduler-ui的目录，执行ui项目的编译，由于是webpack和vue，所以需要以下命令：</p>
+<pre><code>1. npm install. #没有npm的，mac用brew安装一个。brew install npm。
+2. npm run build. #执行完build命令后，会生成dist文件夹，这个文件夹一定要和nginx配置文件的40行所指的目录相同。
+</code></pre>
+<p>保存以下内容到dolphinscheduler.conf文件：</p>
+<pre><code>
+#user nobody;
+worker_processes 1;
+
+error_log /usr/local/etc/nginx/logs/error.log;
+error_log /usr/local/etc/nginx/logs/error.log notice;
+error_log /usr/local/etc/nginx/logs/error.log info;
+
+pid /usr/local/etc/nginx/logs/nginx.pid;
+
+
+events {
+ worker_connections 1024;
+}
+
+
+http {
+ include mime.types;
+ default_type application/octet-stream;
+
+ #log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+ # '$status $body_bytes_sent "$http_referer" '
+ # '"$http_user_agent" "$http_x_forwarded_for"';
+
+ access_log /usr/local/etc/nginx/logs/access.log;
+
+ sendfile on;
+ #tcp_nopush on;
+
+ #keepalive_timeout 0;
+ keepalive_timeout 65;
+
+ #gzip on;
+ server {
+ listen 8888;# access port
+ server_name localhost;
+ #charset koi8-r;
+ access_log /usr/local/etc/nginx/logs/host.access.log;
+ location / {
+ root /xxx/xxx/incubator-dolphinscheduler/dolphinscheduler-ui/dist; #这里为ui目录，需要修改
+ index index.html index.html;
+ }
+ location /dolphinscheduler {
+ proxy_pass http://localhost:12345; # interface address
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header x_real_ipP $remote_addr;
+ proxy_set_header remote_addr $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_http_version 1.1;
+ proxy_connect_timeout 300s;
+ proxy_read_timeout 300s;
+ proxy_send_timeout 300s;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection upgrade;
+ }
+ #error_page 404 /404.html;
+ # redirect server error pages to the static page /50x.html
+ #
+ error_page 500 502 503 504 /50x.html;
+ location = /50x.html {
+ root /usr/share/nginx/html;
+ }
+ }
+ include servers/*;
+}
+
+</code></pre>
+<p>修改上面root指向，然后将文件放到nginx配置同级目录,使用以下命令，启动nginx。</p>
+<pre><code>./nginx -c dolphinscheduler.conf
+</code></pre>
+<p>启动ApiApplicationServer类：
+org.apache.dolphinscheduler.api.ApiApplicationServer类main函数增加如下代码：</p>
+<pre><code>System.setProperty("spring.profiles.active", "api");
+</code></pre>
+<p>修改api模块resources目录下apiserver_logback.xml文件，增加以下代码：</p>
+<pre><code><root level="INFO">
+ <appender-ref ref="APISERVERLOGFILE" />
+ <!-- 增加日志到控制台-->
+ <appender-ref ref="STDOUT"/>
+</root>
+</code></pre>
+<pre><code>执行ApiApplicationServer, 然后访问localhost:8888，账号admin/dolphinscheduler123。
+</code></pre>
+<h4>资源中心</h4>
+<p>ds对于资源的存储在HDFS/S3中，HDFS的mac安装请参考:<a href="https://www.jianshu.com/p/935b4c5e4c25">https://www.jianshu.com/p/935b4c5e4c25</a>
+mac的hdfs请下载:<a href="https://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz">https://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz</a>
+安装好hdfs后，修改common.properties配置：</p>
+<pre><code>res.upload.startup.type=HDFS
+</code></pre>
+<p>修改hadoop.properties：</p>
+<pre><code>fs.defaultFS=hdfs://localhost:9000
+</code></pre>
+<p>然后重启ApiApplicationServer即可。</p>
+<p>*<code>注意</code>：上传的文件大小，超过1M下，需要添加nginx参数</p>
+<pre><code>client_max_body_size 100m;
+</code></pre>
+<h2>DS中概念解释</h2>
+<h4>安全中心</h4>
+<ol>
+<li>租户管理</li>
+</ol>
+<ul>
+<li>作用：对应Linux的用户，当Linux没有对应用户时，将创建用户用于worker执行任务。</li>
+<li>创建：需要注意，创建租户时，【租户编码】为对应的Linux用户。</li>
+</ul>
+<ol start="2">
+<li>用户管理</li>
+</ol>
+<ul>
+<li>作用：创建不同的用户，赋予不同的模块权限。</li>
+<li>创建：新建的用户都为普通用户，需要设置租户，队列，邮件和手机。其中，邮件和手机为告警时使用。</li>
+<li>更新：当更新用户的租户时，相应的资源文件也会更新到新的租户下。</li>
+</ul>
+<ol start="3">
+<li>告警组管理</li>
+</ol>
+<ul>
+<li>作用：创建不同的告警组，添加不同的组员。根据任务运行时的告警配置，当任务执行成功或失败等状态时，进行相应的告警。</li>
+</ul>
+<ol start="4">
+<li>队列管理</li>
+</ol>
+<ul>
+<li>作用：创建Spark，MapReduce，Flink需要用到队列。</li>
+<li>创建：队列的创建，必须名称和值都唯一才可以。</li>
+</ul>
+<ol start="5">
+<li>Worker分组管理</li>
+</ol>
+<ul>
+<li>作用：任务只被组内所指定的worker执行。当新建任务时，选择的【Worker分组】为【Default】时，任务可在任何一台worker上执行。</li>
+</ul>
+<ol start="6">
+<li>令牌管理</li>
+</ol>
+<ul>
+<li>作用：访问后端接口，可以使用系统令牌。</li>
+<li>使用方式：</li>
+</ul>
+<pre><code>HttpPost httpPost = new HttpPost("http://127.0.0.1:12345/escheduler/projects/create");
+httpPost.setHeader("token", "123");
+</code></pre>
+</div></section><footer class="footer-container"><div class="footer-body"><img src="/img/ds_gray.svg"/><div class="cols-container"><div class="col col-12"><h3>Disclaimer</h3><p>Apache DolphinScheduler (incubating) is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by Incubator.
+Incubation is required of all newly accepted projects until a further review indicates
+that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects.
+While incubation status is not necessarily a reflection of the completeness or stability of the code,
+it does indicate that the project has yet to be fully endorsed by the ASF.</p></div><div class="col col-6"><dl><dt>文档</dt><dd><a href="/zh-cn/docs/developer_guide/architecture-design.html" target="_self">概览</a></dd><dd><a href="/zh-cn/docs/1.2.0/user_doc/quick-start.html" target="_self">快速开始</a></dd><dd><a href="/zh-cn/docs/development/developers.html" target="_self">开发者指南</a></dd></dl></div><div class="col col-6"><dl><dt>ASF</dt><dd><a href="http://www.apache.org" target="_self">基金会</a> [...]
+ <script src="https://f.alicdn.com/react/15.4.1/react-with-addons.min.js"></script>
+ <script src="https://f.alicdn.com/react/15.4.1/react-dom.min.js"></script>
+ <script>
+ window.rootPath = '';
+ </script>
+ <script src="/build/documentation.js"></script>
+</body>
+</html>
\ No newline at end of file
diff --git a/zh-cn/docs/developer_guide/dolphin-scheduler-setup.json b/zh-cn/docs/developer_guide/dolphin-scheduler-setup.json
new file mode 100644
index 0000000..b33817c
--- /dev/null
+++ b/zh-cn/docs/developer_guide/dolphin-scheduler-setup.json
@@ -0,0 +1,6 @@
+{
+ "filename": "dolphin-scheduler-setup.md",
+ "__html": "<h4>准备工作</h4>\n<ol>\n<li>fork <a href=\"https://github.com/apache/incubator-dolphinscheduler\">dolphinscheduler</a></li>\n<li>git clone xxx，</li>\n<li>安装好MySQL,Zookeeper,Nginx，Hadoop(可选)。</li>\n<li>git clone项目后，进入目录，执行以下命令。</li>\n</ol>\n<pre><code>1. git branch -a #查看分支\n2. git checkout dev-db #切换到dev-db分支\n3. git pull #同步分支\n4. mvn -U clean package assembly:assembly -Dmaven.test.skip=true #编译项目。由于项目前端的日志模块，使用了gRPC调用后端，所以需要先编译项目。\n</code></pre>\n<h4>搭建后端</h4>\n<ol>\n<li [...]
+ "link": "/zh-cn/docs/developer_guide/dolphin-scheduler-setup.html",
+ "meta": {}
+}
\ No newline at end of file