You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2021/06/09 12:36:49 UTC
[hudi] branch asf-site updated: Travis CI build asf-site
This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 142e34d Travis CI build asf-site
142e34d is described below
commit 142e34d12793af7cd75f446dbc89332cda6b49e9
Author: CI <ci...@hudi.apache.org>
AuthorDate: Wed Jun 9 12:36:37 2021 +0000
Travis CI build asf-site
---
content/assets/js/lunr/lunr-store.js | 12 +++-
content/cn/docs/cloud.html | 3 +
content/docs/{cloud.html => bos_hoodie.html} | 94 +++++++++++++++++++---------
content/docs/cloud.html | 2 +
content/sitemap.xml | 8 +++
5 files changed, 87 insertions(+), 32 deletions(-)
diff --git a/content/assets/js/lunr/lunr-store.js b/content/assets/js/lunr/lunr-store.js
index fa70f46..ffeac7d 100644
--- a/content/assets/js/lunr/lunr-store.js
+++ b/content/assets/js/lunr/lunr-store.js
@@ -1429,6 +1429,16 @@ var store = [{
"tags": [],
"url": "https://hudi.apache.org/docs/ibm_cos_hoodie.html",
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
+ "title": "BOS Filesystem",
+ "excerpt":"这个页面描述了如何让你的Hudi任务使用Baidu BOS存储。 Baidu BOS 部署 为了让Hudi使用BOS，需要增加两部分的配置: 为Hudi增加Baidu BOS的相关配置 增加Jar包到classpath Baidu BOS 相关的配置 新增下面的配置到你的Hudi能访问的core-site.xml文件。使用你的BOS bucket name替换掉fs.defaultFS，使用BOS endpoint地址替换fs.bos.endpoint，使用BOS的key和secret分别替换fs.bos.access.key和fs.bos.secret.access.key，这样Hudi就能读写相应的bucket。 <property> <name>fs.defaultFS</name> <value>bos://bucketname/</value> </property> <property> <name>fs.bos.endpoi [...]
+ "tags": [],
+ "url": "https://hudi.apache.org/docs/bos_hoodie.html",
+ "teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
+ "title": "BOS Filesystem",
+ "excerpt":"In this page, we explain how to get your Hudi job to store into Baidu BOS. Baidu BOS configs There are two configurations required for Hudi-BOS compatibility: Adding Baidu BOS Credentials for Hudi Adding required Jars to classpath Baidu BOS Credentials Add the required configs in your core-site.xml from where...","categories": [],
+ "tags": [],
+ "url": "https://hudi.apache.org/docs/bos_hoodie.html",
+ "teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
"title": "Quick-Start Guide",
"excerpt":"本指南通过使用spark-shell简要介绍了Hudi功能。使用Spark数据源，我们将通过代码段展示如何插入和更新Hudi的默认存储类型数据集： 写时复制。每次写操作之后，我们还将展示如何读取快照和增量数据。 设置spark-shell Hudi适用于Spark-2.4.3+ & Spark 3.x 版本。您可以按照此处的说明设置spark。 在提取的目录中，使用spark-shell运行Hudi： // spark-shell for spark 3 spark-shell \\ --packages org.apache.hudi:hudi-spark3-bundle_2.12:0.8.0,org.apache.spark:spark-avro_2.12:3.0.1 \\ --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' // spark-shell for spark 2 with scala 2.12 spark-shell \ [...]
"tags": [],
@@ -1540,7 +1550,7 @@ var store = [{
"url": "https://hudi.apache.org/docs/deployment.html",
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
"title": "云储存",
- "excerpt":"与云存储连接 无论使用RDD/WriteClient API还是数据源，以下信息都有助于配置对云存储的访问。 AWS S3 S3和Hudi协同工作所需的配置。 Google Cloud Storage GCS和Hudi协同工作所需的配置。 Alibaba Cloud OSS 阿里云和Hudi协同工作所需的配置。 Microsoft Azure Azure和Hudi协同工作所需的配置。 Tencent Cloud Object Storage COS和Hudi协同工作所需的配置。 IBM Cloud Object Storage IBM Cloud Object Storage和Hudi协同工作所需的配置。 ","categories": [],
+ "excerpt":"与云存储连接 无论使用RDD/WriteClient API还是数据源，以下信息都有助于配置对云存储的访问。 AWS S3 S3和Hudi协同工作所需的配置。 Google Cloud Storage GCS和Hudi协同工作所需的配置。 Alibaba Cloud OSS 阿里云和Hudi协同工作所需的配置。 Microsoft Azure Azure和Hudi协同工作所需的配置。 Tencent Cloud Object Storage COS和Hudi协同工作所需的配置。 IBM Cloud Object Storage IBM Cloud Object Storage和Hudi协同工作所需的配置。 Baidu Cloud Object Storage 百度BOS和Hudi协同工作所需的配置。 ","categories": [],
"tags": [],
"url": "https://hudi.apache.org/cn/docs/cloud.html",
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
diff --git a/content/cn/docs/cloud.html b/content/cn/docs/cloud.html
index 885cf08..20398a2 100644
--- a/content/cn/docs/cloud.html
+++ b/content/cn/docs/cloud.html
@@ -379,8 +379,11 @@ Azure和Hudi协同工作所需的配置。</li>
COS和Hudi协同工作所需的配置。</li>
<li><a href="/cn/docs/ibm_cos_hoodie.html">IBM Cloud Object Storage</a> <br />
IBM Cloud Object Storage和Hudi协同工作所需的配置。</li>
+ <li><a href="/docs/bos_hoodie.html">Baidu Cloud Object Storage</a> <br />
+ 百度BOS和Hudi协同工作所需的配置。</li>
</ul>
+
</section>
<a href="#masthead__inner-wrap" class="back-to-top">Back to top ↑</a>
diff --git a/content/docs/cloud.html b/content/docs/bos_hoodie.html
similarity index 69%
copy from content/docs/cloud.html
copy to content/docs/bos_hoodie.html
index 712ec89..6e05606 100644
--- a/content/docs/cloud.html
+++ b/content/docs/bos_hoodie.html
@@ -3,23 +3,23 @@
<head>
<meta charset="utf-8">
-<!-- begin _includes/seo.html --><title>Cloud Storage - Apache Hudi</title>
-<meta name="description" content="Talking to Cloud Storage">
+<!-- begin _includes/seo.html --><title>BOS Filesystem - Apache Hudi</title>
+<meta name="description" content="In this page, we explain how to get your Hudi job to store into Baidu BOS.">
<meta property="og:type" content="article">
<meta property="og:locale" content="en_US">
<meta property="og:site_name" content="">
-<meta property="og:title" content="Cloud Storage">
-<meta property="og:url" content="https://hudi.apache.org/docs/cloud.html">
+<meta property="og:title" content="BOS Filesystem">
+<meta property="og:url" content="https://hudi.apache.org/docs/bos_hoodie.html">
- <meta property="og:description" content="Talking to Cloud Storage">
+ <meta property="og:description" content="In this page, we explain how to get your Hudi job to store into Baidu BOS.">
- <meta property="article:modified_time" content="2019-06-16T21:59:57-04:00">
+ <meta property="article:modified_time" content="2021-06-09T17:38:24-04:00">
@@ -288,7 +288,7 @@
- <li><a href="/docs/cloud.html" class="active">Storage Configuration</a></li>
+ <li><a href="/docs/cloud.html" class="">Storage Configuration</a></li>
@@ -345,7 +345,7 @@
<div class="page__inner-wrap">
<header>
- <h1 id="page-title" class="page__title" itemprop="headline">Cloud Storage
+ <h1 id="page-title" class="page__title" itemprop="headline">BOS Filesystem
</h1>
<!-- Output author details if some exist. -->
@@ -354,35 +354,67 @@
<section class="page__content" itemprop="text">
- <aside class="sidebar__right sticky">
- <nav class="toc">
- <header><h4 class="nav__title"><i class="fas fa-file-alt"></i> IN THIS PAGE</h4></header>
- <ul class="toc__menu">
- <li><a href="#talking-to-cloud-storage">Talking to Cloud Storage</a></li>
-</ul>
- </nav>
- </aside>
+ <style>
+ .page {
+ padding-right: 0 !important;
+ }
+ </style>
- <h2 id="talking-to-cloud-storage">Talking to Cloud Storage</h2>
+ <p>In this page, we explain how to get your Hudi job to store into Baidu BOS.</p>
+
+<h2 id="baidu-bos-configs">Baidu BOS configs</h2>
-<p>Immaterial of whether RDD/WriteClient APIs or Datasource is used, the following information helps configure access
-to cloud stores.</p>
+<p>There are two configurations required for Hudi-BOS compatibility:</p>
<ul>
- <li><a href="/docs/s3_hoodie">AWS S3</a> <br />
-Configurations required for S3 and Hudi co-operability.</li>
- <li><a href="/docs/gcs_hoodie">Google Cloud Storage</a> <br />
-Configurations required for GCS and Hudi co-operability.</li>
- <li><a href="/docs/oss_hoodie.html">Alibaba Cloud OSS</a> <br />
-Configurations required for OSS and Hudi co-operability.</li>
- <li><a href="/docs/azure_hoodie.html">Microsoft Azure</a> <br />
-Configurations required for Azure and Hudi co-operability.</li>
- <li><a href="/docs/cos_hoodie.html">Tencent Cloud Object Storage</a> <br />
- Configurations required for COS and Hudi co-operability.</li>
- <li><a href="/docs/ibm_cos_hoodie.html">IBM Cloud Object Storage</a> <br />
- Configurations required for IBM Cloud Object Storage and Hudi co-operability.</li>
+ <li>Adding Baidu BOS Credentials for Hudi</li>
+ <li>Adding required Jars to classpath</li>
</ul>
+<h3 id="baidu-bos-credentials">Baidu BOS Credentials</h3>
+
+<p>Add the required configs in your core-site.xml from where Hudi can fetch them. Replace the <code class="highlighter-rouge">fs.defaultFS</code> with your BOS bucket name, replace <code class="highlighter-rouge">fs.bos.endpoint</code> with your bos endpoint, replace <code class="highlighter-rouge">fs.bos.access.key</code> with your bos key, replace <code class="highlighter-rouge">fs.bos.secret.access.key</code> with your bos secret key. Hudi should be able to read/write from the bucket.</p>
+
+<div class="language-xml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt"><property></span>
+ <span class="nt"><name></span>fs.defaultFS<span class="nt"></name></span>
+ <span class="nt"><value></span>bos://bucketname/<span class="nt"></value></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>fs.bos.endpoint<span class="nt"></name></span>
+ <span class="nt"><value></span>bos-endpoint-address<span class="nt"></value></span>
+ <span class="nt"><description></span>Baidu bos endpoint to connect to,for example : http://bj.bcebos.com<span class="nt"></description></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>fs.bos.access.key<span class="nt"></name></span>
+ <span class="nt"><value></span>bos-key<span class="nt"></value></span>
+ <span class="nt"><description></span>Baidu access key<span class="nt"></description></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>fs.bos.secret.access.key<span class="nt"></name></span>
+ <span class="nt"><value></span>bos-secret-key<span class="nt"></value></span>
+ <span class="nt"><description></span>Baidu secret key.<span class="nt"></description></span>
+<span class="nt"></property></span>
+
+<span class="nt"><property></span>
+ <span class="nt"><name></span>fs.bos.impl<span class="nt"></name></span>
+ <span class="nt"><value></span>org.apache.hadoop.fs.bos.BaiduBosFileSystem<span class="nt"></value></span>
+<span class="nt"></property></span>
+</code></pre></div></div>
+
+<h3 id="baidu-bos-libs">Baidu bos Libs</h3>
+
+<p>Baidu hadoop libraries jars to add to our classpath</p>
+
+<ul>
+ <li>com.baidubce:bce-java-sdk:0.10.165</li>
+ <li>bos-hdfs-sdk-1.0.2-community.jar</li>
+</ul>
+
+<p>You can download the bos-hdfs-sdk jar from <a href="https://sdk.bce.baidu.com/console-sdk/bos-hdfs-sdk-1.0.2-community.jar.zip">here</a> , and then unzip it.</p>
+
</section>
<a href="#masthead__inner-wrap" class="back-to-top">Back to top ↑</a>
diff --git a/content/docs/cloud.html b/content/docs/cloud.html
index 712ec89..096f4b8 100644
--- a/content/docs/cloud.html
+++ b/content/docs/cloud.html
@@ -381,6 +381,8 @@ Configurations required for Azure and Hudi co-operability.</li>
Configurations required for COS and Hudi co-operability.</li>
<li><a href="/docs/ibm_cos_hoodie.html">IBM Cloud Object Storage</a> <br />
Configurations required for IBM Cloud Object Storage and Hudi co-operability.</li>
+ <li><a href="/docs/bos_hoodie.html">Baidu Cloud Object Storage</a> <br />
+ Configurations required for BOS and Hudi co-operability.</li>
</ul>
</section>
diff --git a/content/sitemap.xml b/content/sitemap.xml
index bc02ba6..ded240e 100644
--- a/content/sitemap.xml
+++ b/content/sitemap.xml
@@ -1145,6 +1145,14 @@
<lastmod>2020-10-01T17:38:24-04:00</lastmod>
</url>
<url>
+<loc>https://hudi.apache.org/docs/bos_hoodie.html</loc>
+<lastmod>2021-06-09T17:38:24-04:00</lastmod>
+</url>
+<url>
+<loc>https://hudi.apache.org/docs/bos_hoodie.html</loc>
+<lastmod>2021-06-09T17:38:24-04:00</lastmod>
+</url>
+<url>
<loc>https://hudi.apache.org/cn/docs/spark_quick-start-guide.html</loc>
<lastmod>2019-12-30T14:59:57-05:00</lastmod>
</url>