You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2017/04/24 11:15:29 UTC
[04/23] incubator-hivemall-site git commit: Added userguide contents
about clustering and GeoSpatial
http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/06c5398a/userguide/spark/misc/functions.html
----------------------------------------------------------------------
diff --git a/userguide/spark/misc/functions.html b/userguide/spark/misc/functions.html
new file mode 100644
index 0000000..1e8de0f
--- /dev/null
+++ b/userguide/spark/misc/functions.html
@@ -0,0 +1,2163 @@
+
+<!DOCTYPE HTML>
+<html lang="" >
+ <head>
+ <meta charset="UTF-8">
+ <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
+ <title>Other utility functions � Hivemall User Manual</title>
+ <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+ <meta name="description" content="">
+ <meta name="generator" content="GitBook 3.2.2">
+
+
+
+
+ <link rel="stylesheet" href="../../gitbook/style.css">
+
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-splitter/splitter.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-etoc/plugin.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-callouts/plugin.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-toggle-chapters/toggle.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-codeblock-filename/block.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-multipart/multipart.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-katex/katex.min.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-emphasize/plugin.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-highlight/website.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-search/search.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-fontsettings/website.css">
+
+
+
+ <link rel="stylesheet" href="../../gitbook/gitbook-plugin-theme-api/theme-api.css">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <meta name="HandheldFriendly" content="true"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
+ <meta name="apple-mobile-web-app-capable" content="yes">
+ <meta name="apple-mobile-web-app-status-bar-style" content="black">
+ <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
+ <link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
+
+
+
+ <link rel="prev" href="topk_join.html" />
+
+
+ </head>
+ <body>
+
+<div class="book">
+ <div class="book-summary">
+
+
+<div id="book-search-input" role="search">
+ <input type="text" placeholder="Type to search" />
+</div>
+
+
+ <nav role="navigation">
+
+
+
+<ul class="summary">
+
+
+
+
+ <li>
+ <a href="http://hivemall.incubator.apache.org/" target="_blank" class="custom-link"><i class="fa fa-home"></i> Home</a>
+ </li>
+
+
+
+
+ <li class="divider"></li>
+
+
+
+
+ <li class="header">TABLE OF CONTENTS</li>
+
+
+
+ <li class="chapter " data-level="1.1" data-path="../../">
+
+ <a href="../../">
+
+
+ <b>1.1.</b>
+
+ Introduction
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.2" data-path="../../getting_started/">
+
+ <a href="../../getting_started/">
+
+
+ <b>1.2.</b>
+
+ Getting Started
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="1.2.1" data-path="../../getting_started/installation.html">
+
+ <a href="../../getting_started/installation.html">
+
+
+ <b>1.2.1.</b>
+
+ Installation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.2.2" data-path="../../getting_started/permanent-functions.html">
+
+ <a href="../../getting_started/permanent-functions.html">
+
+
+ <b>1.2.2.</b>
+
+ Install as permanent functions
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.2.3" data-path="../../getting_started/input-format.html">
+
+ <a href="../../getting_started/input-format.html">
+
+
+ <b>1.2.3.</b>
+
+ Input Format
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="1.3" data-path="../../tips/">
+
+ <a href="../../tips/">
+
+
+ <b>1.3.</b>
+
+ Tips for Effective Hivemall
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="1.3.1" data-path="../../tips/addbias.html">
+
+ <a href="../../tips/addbias.html">
+
+
+ <b>1.3.1.</b>
+
+ Explicit addBias() for better prediction
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.2" data-path="../../tips/rand_amplify.html">
+
+ <a href="../../tips/rand_amplify.html">
+
+
+ <b>1.3.2.</b>
+
+ Use rand_amplify() to better prediction results
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.3" data-path="../../tips/rt_prediction.html">
+
+ <a href="../../tips/rt_prediction.html">
+
+
+ <b>1.3.3.</b>
+
+ Real-time Prediction on RDBMS
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.4" data-path="../../tips/ensemble_learning.html">
+
+ <a href="../../tips/ensemble_learning.html">
+
+
+ <b>1.3.4.</b>
+
+ Ensemble learning for stable prediction
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.5" data-path="../../tips/mixserver.html">
+
+ <a href="../../tips/mixserver.html">
+
+
+ <b>1.3.5.</b>
+
+ Mixing models for a better prediction convergence (MIX server)
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.6" data-path="../../tips/emr.html">
+
+ <a href="../../tips/emr.html">
+
+
+ <b>1.3.6.</b>
+
+ Run Hivemall on Amazon Elastic MapReduce
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="1.4" data-path="../../tips/general_tips.html">
+
+ <a href="../../tips/general_tips.html">
+
+
+ <b>1.4.</b>
+
+ General Hive/Hadoop tips
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="1.4.1" data-path="../../tips/rowid.html">
+
+ <a href="../../tips/rowid.html">
+
+
+ <b>1.4.1.</b>
+
+ Adding rowid for each row
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.4.2" data-path="../../tips/hadoop_tuning.html">
+
+ <a href="../../tips/hadoop_tuning.html">
+
+
+ <b>1.4.2.</b>
+
+ Hadoop tuning for Hivemall
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="1.5" data-path="../../troubleshooting/">
+
+ <a href="../../troubleshooting/">
+
+
+ <b>1.5.</b>
+
+ Troubleshooting
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="1.5.1" data-path="../../troubleshooting/oom.html">
+
+ <a href="../../troubleshooting/oom.html">
+
+
+ <b>1.5.1.</b>
+
+ OutOfMemoryError in training
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.5.2" data-path="../../troubleshooting/mapjoin_task_error.html">
+
+ <a href="../../troubleshooting/mapjoin_task_error.html">
+
+
+ <b>1.5.2.</b>
+
+ SemanticException Generate Map Join Task Error: Cannot serialize object
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.5.3" data-path="../../troubleshooting/asterisk.html">
+
+ <a href="../../troubleshooting/asterisk.html">
+
+
+ <b>1.5.3.</b>
+
+ Asterisk argument for UDTF does not work
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.5.4" data-path="../../troubleshooting/num_mappers.html">
+
+ <a href="../../troubleshooting/num_mappers.html">
+
+
+ <b>1.5.4.</b>
+
+ The number of mappers is less than input splits in Hadoop 2.x
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.5.5" data-path="../../troubleshooting/mapjoin_classcastex.html">
+
+ <a href="../../troubleshooting/mapjoin_classcastex.html">
+
+
+ <b>1.5.5.</b>
+
+ Map-side Join causes ClassCastException on Tez
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part II - Generic Features</li>
+
+
+
+ <li class="chapter " data-level="2.1" data-path="../../misc/generic_funcs.html">
+
+ <a href="../../misc/generic_funcs.html">
+
+
+ <b>2.1.</b>
+
+ List of generic Hivemall functions
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="2.2" data-path="../../misc/topk.html">
+
+ <a href="../../misc/topk.html">
+
+
+ <b>2.2.</b>
+
+ Efficient Top-K query processing
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="2.3" data-path="../../misc/tokenizer.html">
+
+ <a href="../../misc/tokenizer.html">
+
+
+ <b>2.3.</b>
+
+ English/Japanese Text Tokenizer
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part III - Feature Engineering</li>
+
+
+
+ <li class="chapter " data-level="3.1" data-path="../../ft_engineering/scaling.html">
+
+ <a href="../../ft_engineering/scaling.html">
+
+
+ <b>3.1.</b>
+
+ Feature Scaling
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="3.2" data-path="../../ft_engineering/hashing.html">
+
+ <a href="../../ft_engineering/hashing.html">
+
+
+ <b>3.2.</b>
+
+ Feature Hashing
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="3.3" data-path="../../ft_engineering/tfidf.html">
+
+ <a href="../../ft_engineering/tfidf.html">
+
+
+ <b>3.3.</b>
+
+ TF-IDF calculation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="3.4" data-path="../../ft_engineering/ft_trans.html">
+
+ <a href="../../ft_engineering/ft_trans.html">
+
+
+ <b>3.4.</b>
+
+ FEATURE TRANSFORMATION
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="3.4.1" data-path="../../ft_engineering/vectorizer.html">
+
+ <a href="../../ft_engineering/vectorizer.html">
+
+
+ <b>3.4.1.</b>
+
+ Vectorize Features
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="3.4.2" data-path="../../ft_engineering/quantify.html">
+
+ <a href="../../ft_engineering/quantify.html">
+
+
+ <b>3.4.2.</b>
+
+ Quantify non-number features
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="3.5" data-path="../../ft_engineering/feature_selection.html">
+
+ <a href="../../ft_engineering/feature_selection.html">
+
+
+ <b>3.5.</b>
+
+ Feature selection
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part IV - Evaluation</li>
+
+
+
+ <li class="chapter " data-level="4.1" data-path="../../eval/stat_eval.html">
+
+ <a href="../../eval/stat_eval.html">
+
+
+ <b>4.1.</b>
+
+ Statistical evaluation of a prediction model
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="4.1.1" data-path="../../eval/auc.html">
+
+ <a href="../../eval/auc.html">
+
+
+ <b>4.1.1.</b>
+
+ Area Under the ROC Curve
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="4.2" data-path="../../eval/rank.html">
+
+ <a href="../../eval/rank.html">
+
+
+ <b>4.2.</b>
+
+ Ranking Measures
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="4.3" data-path="../../eval/datagen.html">
+
+ <a href="../../eval/datagen.html">
+
+
+ <b>4.3.</b>
+
+ Data Generation
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="4.3.1" data-path="../../eval/lr_datagen.html">
+
+ <a href="../../eval/lr_datagen.html">
+
+
+ <b>4.3.1.</b>
+
+ Logistic Regression data generation
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part V - Binary classification</li>
+
+
+
+ <li class="chapter " data-level="5.1" data-path="../../binaryclass/a9a.html">
+
+ <a href="../../binaryclass/a9a.html">
+
+
+ <b>5.1.</b>
+
+ a9a Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.1.1" data-path="../../binaryclass/a9a_dataset.html">
+
+ <a href="../../binaryclass/a9a_dataset.html">
+
+
+ <b>5.1.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.1.2" data-path="../../binaryclass/a9a_lr.html">
+
+ <a href="../../binaryclass/a9a_lr.html">
+
+
+ <b>5.1.2.</b>
+
+ Logistic Regression
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.1.3" data-path="../../binaryclass/a9a_minibatch.html">
+
+ <a href="../../binaryclass/a9a_minibatch.html">
+
+
+ <b>5.1.3.</b>
+
+ Mini-batch Gradient Descent
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="5.2" data-path="../../binaryclass/news20.html">
+
+ <a href="../../binaryclass/news20.html">
+
+
+ <b>5.2.</b>
+
+ News20 Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.2.1" data-path="../../binaryclass/news20_dataset.html">
+
+ <a href="../../binaryclass/news20_dataset.html">
+
+
+ <b>5.2.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.2.2" data-path="../../binaryclass/news20_pa.html">
+
+ <a href="../../binaryclass/news20_pa.html">
+
+
+ <b>5.2.2.</b>
+
+ Perceptron, Passive Aggressive
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.2.3" data-path="../../binaryclass/news20_scw.html">
+
+ <a href="../../binaryclass/news20_scw.html">
+
+
+ <b>5.2.3.</b>
+
+ CW, AROW, SCW
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.2.4" data-path="../../binaryclass/news20_adagrad.html">
+
+ <a href="../../binaryclass/news20_adagrad.html">
+
+
+ <b>5.2.4.</b>
+
+ AdaGradRDA, AdaGrad, AdaDelta
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="5.3" data-path="../../binaryclass/kdd2010a.html">
+
+ <a href="../../binaryclass/kdd2010a.html">
+
+
+ <b>5.3.</b>
+
+ KDD2010a Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.3.1" data-path="../../binaryclass/kdd2010a_dataset.html">
+
+ <a href="../../binaryclass/kdd2010a_dataset.html">
+
+
+ <b>5.3.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.3.2" data-path="../../binaryclass/kdd2010a_scw.html">
+
+ <a href="../../binaryclass/kdd2010a_scw.html">
+
+
+ <b>5.3.2.</b>
+
+ PA, CW, AROW, SCW
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="5.4" data-path="../../binaryclass/kdd2010b.html">
+
+ <a href="../../binaryclass/kdd2010b.html">
+
+
+ <b>5.4.</b>
+
+ KDD2010b Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.4.1" data-path="../../binaryclass/kdd2010b_dataset.html">
+
+ <a href="../../binaryclass/kdd2010b_dataset.html">
+
+
+ <b>5.4.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.4.2" data-path="../../binaryclass/kdd2010b_arow.html">
+
+ <a href="../../binaryclass/kdd2010b_arow.html">
+
+
+ <b>5.4.2.</b>
+
+ AROW
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="5.5" data-path="../../binaryclass/webspam.html">
+
+ <a href="../../binaryclass/webspam.html">
+
+
+ <b>5.5.</b>
+
+ Webspam Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.5.1" data-path="../../binaryclass/webspam_dataset.html">
+
+ <a href="../../binaryclass/webspam_dataset.html">
+
+
+ <b>5.5.1.</b>
+
+ Data pareparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.5.2" data-path="../../binaryclass/webspam_scw.html">
+
+ <a href="../../binaryclass/webspam_scw.html">
+
+
+ <b>5.5.2.</b>
+
+ PA1, AROW, SCW
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="5.6" data-path="../../binaryclass/titanic_rf.html">
+
+ <a href="../../binaryclass/titanic_rf.html">
+
+
+ <b>5.6.</b>
+
+ Kaggle Titanic Tutorial
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part VI - Multiclass classification</li>
+
+
+
+ <li class="chapter " data-level="6.1" data-path="../../multiclass/news20.html">
+
+ <a href="../../multiclass/news20.html">
+
+
+ <b>6.1.</b>
+
+ News20 Multiclass Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="6.1.1" data-path="../../multiclass/news20_dataset.html">
+
+ <a href="../../multiclass/news20_dataset.html">
+
+
+ <b>6.1.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.2" data-path="../../multiclass/news20_one-vs-the-rest_dataset.html">
+
+ <a href="../../multiclass/news20_one-vs-the-rest_dataset.html">
+
+
+ <b>6.1.2.</b>
+
+ Data preparation for one-vs-the-rest classifiers
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.3" data-path="../../multiclass/news20_pa.html">
+
+ <a href="../../multiclass/news20_pa.html">
+
+
+ <b>6.1.3.</b>
+
+ PA
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.4" data-path="../../multiclass/news20_scw.html">
+
+ <a href="../../multiclass/news20_scw.html">
+
+
+ <b>6.1.4.</b>
+
+ CW, AROW, SCW
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.5" data-path="../../multiclass/news20_ensemble.html">
+
+ <a href="../../multiclass/news20_ensemble.html">
+
+
+ <b>6.1.5.</b>
+
+ Ensemble learning
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.6" data-path="../../multiclass/news20_one-vs-the-rest.html">
+
+ <a href="../../multiclass/news20_one-vs-the-rest.html">
+
+
+ <b>6.1.6.</b>
+
+ one-vs-the-rest classifier
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="6.2" data-path="../../multiclass/iris.html">
+
+ <a href="../../multiclass/iris.html">
+
+
+ <b>6.2.</b>
+
+ Iris Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="6.2.1" data-path="../../multiclass/iris_dataset.html">
+
+ <a href="../../multiclass/iris_dataset.html">
+
+
+ <b>6.2.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.2.2" data-path="../../multiclass/iris_scw.html">
+
+ <a href="../../multiclass/iris_scw.html">
+
+
+ <b>6.2.2.</b>
+
+ SCW
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.2.3" data-path="../../multiclass/iris_randomforest.html">
+
+ <a href="../../multiclass/iris_randomforest.html">
+
+
+ <b>6.2.3.</b>
+
+ RandomForest
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part VII - Regression</li>
+
+
+
+ <li class="chapter " data-level="7.1" data-path="../../regression/e2006.html">
+
+ <a href="../../regression/e2006.html">
+
+
+ <b>7.1.</b>
+
+ E2006-tfidf regression Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="7.1.1" data-path="../../regression/e2006_dataset.html">
+
+ <a href="../../regression/e2006_dataset.html">
+
+
+ <b>7.1.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="7.1.2" data-path="../../regression/e2006_arow.html">
+
+ <a href="../../regression/e2006_arow.html">
+
+
+ <b>7.1.2.</b>
+
+ Passive Aggressive, AROW
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="7.2" data-path="../../regression/kddcup12tr2.html">
+
+ <a href="../../regression/kddcup12tr2.html">
+
+
+ <b>7.2.</b>
+
+ KDDCup 2012 track 2 CTR prediction Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="7.2.1" data-path="../../regression/kddcup12tr2_dataset.html">
+
+ <a href="../../regression/kddcup12tr2_dataset.html">
+
+
+ <b>7.2.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="7.2.2" data-path="../../regression/kddcup12tr2_lr.html">
+
+ <a href="../../regression/kddcup12tr2_lr.html">
+
+
+ <b>7.2.2.</b>
+
+ Logistic Regression, Passive Aggressive
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="7.2.3" data-path="../../regression/kddcup12tr2_lr_amplify.html">
+
+ <a href="../../regression/kddcup12tr2_lr_amplify.html">
+
+
+ <b>7.2.3.</b>
+
+ Logistic Regression with Amplifier
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="7.2.4" data-path="../../regression/kddcup12tr2_adagrad.html">
+
+ <a href="../../regression/kddcup12tr2_adagrad.html">
+
+
+ <b>7.2.4.</b>
+
+ AdaGrad, AdaDelta
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part VIII - Recommendation</li>
+
+
+
+ <li class="chapter " data-level="8.1" data-path="../../recommend/cf.html">
+
+ <a href="../../recommend/cf.html">
+
+
+ <b>8.1.</b>
+
+ Collaborative Filtering
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="8.1.1" data-path="../../recommend/item_based_cf.html">
+
+ <a href="../../recommend/item_based_cf.html">
+
+
+ <b>8.1.1.</b>
+
+ Item-based Collaborative Filtering
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="8.2" data-path="../../recommend/news20.html">
+
+ <a href="../../recommend/news20.html">
+
+
+ <b>8.2.</b>
+
+ News20 related article recommendation Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="8.2.1" data-path="../../multiclass/news20_dataset.html">
+
+ <a href="../../multiclass/news20_dataset.html">
+
+
+ <b>8.2.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.2.2" data-path="../../recommend/news20_jaccard.html">
+
+ <a href="../../recommend/news20_jaccard.html">
+
+
+ <b>8.2.2.</b>
+
+ LSH/Minhash and Jaccard Similarity
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.2.3" data-path="../../recommend/news20_knn.html">
+
+ <a href="../../recommend/news20_knn.html">
+
+
+ <b>8.2.3.</b>
+
+ LSH/Minhash and Brute-Force Search
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.2.4" data-path="../../recommend/news20_bbit_minhash.html">
+
+ <a href="../../recommend/news20_bbit_minhash.html">
+
+
+ <b>8.2.4.</b>
+
+ kNN search using b-Bits Minhash
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="8.3" data-path="../../recommend/movielens.html">
+
+ <a href="../../recommend/movielens.html">
+
+
+ <b>8.3.</b>
+
+ MovieLens movie recommendation Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="8.3.1" data-path="../../recommend/movielens_dataset.html">
+
+ <a href="../../recommend/movielens_dataset.html">
+
+
+ <b>8.3.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.3.2" data-path="../../recommend/movielens_mf.html">
+
+ <a href="../../recommend/movielens_mf.html">
+
+
+ <b>8.3.2.</b>
+
+ Matrix Factorization
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.3.3" data-path="../../recommend/movielens_fm.html">
+
+ <a href="../../recommend/movielens_fm.html">
+
+
+ <b>8.3.3.</b>
+
+ Factorization Machine
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.3.4" data-path="../../recommend/movielens_cv.html">
+
+ <a href="../../recommend/movielens_cv.html">
+
+
+ <b>8.3.4.</b>
+
+ 10-fold Cross Validation (Matrix Factorization)
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part IX - Anomaly Detection</li>
+
+
+
+ <li class="chapter " data-level="9.1" data-path="../../anomaly/lof.html">
+
+ <a href="../../anomaly/lof.html">
+
+
+ <b>9.1.</b>
+
+ Outlier Detection using Local Outlier Factor (LOF)
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="9.2" data-path="../../anomaly/sst.html">
+
+ <a href="../../anomaly/sst.html">
+
+
+ <b>9.2.</b>
+
+ Change-Point Detection using Singular Spectrum Transformation (SST)
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="9.3" data-path="../../anomaly/changefinder.html">
+
+ <a href="../../anomaly/changefinder.html">
+
+
+ <b>9.3.</b>
+
+ ChangeFinder: Detecting Outlier and Change-Point Simultaneously
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part X - Clustering</li>
+
+
+
+ <li class="chapter " data-level="10.1" data-path="../../clustering/lda.html">
+
+ <a href="../../clustering/lda.html">
+
+
+ <b>10.1.</b>
+
+ Latent Dirichlet Allocation
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part XI - GeoSpatial functions</li>
+
+
+
+ <li class="chapter " data-level="11.1" data-path="../../geospatial/latlon.html">
+
+ <a href="../../geospatial/latlon.html">
+
+
+ <b>11.1.</b>
+
+ Lat/Lon functions
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part XII - Hivemall on Spark</li>
+
+
+
+ <li class="chapter " data-level="12.1" data-path="../getting_started/">
+
+ <a href="../getting_started/">
+
+
+ <b>12.1.</b>
+
+ Getting Started
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="12.1.1" data-path="../getting_started/installation.html">
+
+ <a href="../getting_started/installation.html">
+
+
+ <b>12.1.1.</b>
+
+ Installation
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="12.2" data-path="../binaryclass/">
+
+ <a href="../binaryclass/">
+
+
+ <b>12.2.</b>
+
+ Binary Classification
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="12.2.1" data-path="../binaryclass/a9a_df.html">
+
+ <a href="../binaryclass/a9a_df.html">
+
+
+ <b>12.2.1.</b>
+
+ a9a Tutorial for DataFrame
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="12.3" data-path="../binaryclass/">
+
+ <a href="../binaryclass/">
+
+
+ <b>12.3.</b>
+
+ Regression
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="12.3.1" data-path="../regression/e2006_df.html">
+
+ <a href="../regression/e2006_df.html">
+
+
+ <b>12.3.1.</b>
+
+ E2006-tfidf regression Tutorial for DataFrame
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="12.4" data-path="misc.html">
+
+ <a href="misc.html">
+
+
+ <b>12.4.</b>
+
+ Generic features
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="12.4.1" data-path="topk_join.html">
+
+ <a href="topk_join.html">
+
+
+ <b>12.4.1.</b>
+
+ Top-k Join processing
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter active" data-level="12.4.2" data-path="functions.html">
+
+ <a href="functions.html">
+
+
+ <b>12.4.2.</b>
+
+ Other utility functions
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part XIII - External References</li>
+
+
+
+ <li class="chapter " data-level="13.1" >
+
+ <a target="_blank" href="https://github.com/maropu/hivemall-spark">
+
+
+ <b>13.1.</b>
+
+ Hivemall on Apache Spark
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="13.2" >
+
+ <a target="_blank" href="https://github.com/daijyc/hivemall/wiki/PigHome">
+
+
+ <b>13.2.</b>
+
+ Hivemall on Apache Pig
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="divider"></li>
+
+ <li>
+ <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
+ Published with GitBook
+ </a>
+ </li>
+</ul>
+
+
+ </nav>
+
+
+ </div>
+
+ <div class="book-body">
+
+ <div class="body-inner">
+
+
+
+<div class="book-header" role="navigation">
+
+
+ <!-- Title -->
+ <h1>
+ <i class="fa fa-circle-o-notch fa-spin"></i>
+ <a href="../.." >Other utility functions</a>
+ </h1>
+</div>
+
+
+
+
+ <div class="page-wrapper" tabindex="-1" role="main">
+ <div class="page-inner">
+
+<div id="book-search-results">
+ <div class="search-noresults">
+
+ <section class="normal markdown-section">
+
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<h1 id="flatten">flatten</h1>
+<p><code>df.flatten()</code> flattens a nested schema of <code>df</code> into a flat one.</p>
+<h2 id="usage">Usage</h2>
+<pre><code class="lang-scala">scala> <span class="hljs-keyword">val</span> df = <span class="hljs-type">Seq</span>((<span class="hljs-number">0</span>, (<span class="hljs-number">1</span>, (<span class="hljs-number">3.0</span>, <span class="hljs-string">"a"</span>)), (<span class="hljs-number">5</span>, <span class="hljs-number">0.9</span>))).toDF()
+scala> df.printSchema
+root
+ |-- _1: integer (nullable = <span class="hljs-literal">false</span>)
+ |-- _2: struct (nullable = <span class="hljs-literal">true</span>)
+ | |-- _1: integer (nullable = <span class="hljs-literal">false</span>)
+ | |-- _2: struct (nullable = <span class="hljs-literal">true</span>)
+ | | |-- _1: double (nullable = <span class="hljs-literal">false</span>)
+ | | |-- _2: string (nullable = <span class="hljs-literal">true</span>)
+ |-- _3: struct (nullable = <span class="hljs-literal">true</span>)
+ | |-- _1: integer (nullable = <span class="hljs-literal">false</span>)
+ | |-- _2: double (nullable = <span class="hljs-literal">false</span>)
+
+scala> df.flatten(separator = <span class="hljs-string">"$"</span>).printSchema
+root
+ |-- _1: integer (nullable = <span class="hljs-literal">false</span>)
+ |-- _2$_1: integer (nullable = <span class="hljs-literal">true</span>)
+ |-- _2$_2$_1: double (nullable = <span class="hljs-literal">true</span>)
+ |-- _2$_2$_2: string (nullable = <span class="hljs-literal">true</span>)
+ |-- _3$_1: integer (nullable = <span class="hljs-literal">true</span>)
+ |-- _3$_2: double (nullable = <span class="hljs-literal">true</span>)
+</code></pre>
+<h1 id="fromcsv">from_csv</h1>
+<p>This function parses a column containing a CSV string into a <code>StructType</code>
+with the specified schema.</p>
+<h2 id="usage">Usage</h2>
+<pre><code class="lang-scala">scala> <span class="hljs-keyword">val</span> df = <span class="hljs-type">Seq</span>(<span class="hljs-string">"1, abc, 0.8"</span>).toDF()
+
+scala> df.printSchema
+root
+ |-- value: string (nullable = <span class="hljs-literal">true</span>)
+
+scala> <span class="hljs-keyword">val</span> schema = <span class="hljs-keyword">new</span> <span class="hljs-type">StructType</span>().add(<span class="hljs-string">"a"</span>, <span class="hljs-type">IntegerType</span>).add(<span class="hljs-string">"b"</span>, <span class="hljs-type">StringType</span>).add(<span class="hljs-string">"c"</span>, <span class="hljs-type">DoubleType</span>)
+
+scala> df.select(from_csv($<span class="hljs-string">"value"</span>, schema)).printSchema
+root
+ |-- csvtostruct(value): struct (nullable = <span class="hljs-literal">true</span>)
+ | |-- a: integer (nullable = <span class="hljs-literal">true</span>)
+ | |-- b: string (nullable = <span class="hljs-literal">true</span>)
+ | |-- c: double (nullable = <span class="hljs-literal">true</span>)
+
+scala> df.select(from_csv($<span class="hljs-string">"value"</span>, schema)).show
++------------------+
+|csvtostruct(value)|
++------------------+
+| [<span class="hljs-number">1</span>, abc,<span class="hljs-number">0.8</span>]|
++------------------+
+</code></pre>
+<h1 id="tocsv">to_csv</h1>
+<p>This function converts a column containing a <code>StructType</code> into a CSV string
+with the specified schema.</p>
+<h2 id="usage">Usage</h2>
+<pre><code class="lang-scala">scala> <span class="hljs-keyword">val</span> df = <span class="hljs-type">Seq</span>((<span class="hljs-number">1</span>, <span class="hljs-string">"a"</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3.9</span>, <span class="hljs-string">"abc"</span>)), (<span class="hljs-number">8</span>, <span class="hljs-string">"c"</span>, (<span class="hljs-number">2</span>, <span class="hljs-number">0.4</span>, <span class="hljs-string">"def"</span>))).toDF()
+
+scala> df.printSchema
+root
+ |-- _1: integer (nullable = <span class="hljs-literal">false</span>)
+ |-- _2: string (nullable = <span class="hljs-literal">true</span>)
+ |-- _3: struct (nullable = <span class="hljs-literal">true</span>)
+ | |-- _1: integer (nullable = <span class="hljs-literal">false</span>)
+ | |-- _2: double (nullable = <span class="hljs-literal">false</span>)
+ | |-- _3: string (nullable = <span class="hljs-literal">true</span>)
+
+scala> df.select(to_csv($<span class="hljs-string">"_3"</span>))
+
+scala> df.select(to_csv($<span class="hljs-string">"_3"</span>)).printSchema
+root
+ |-- structtocsv(_3): string (nullable = <span class="hljs-literal">true</span>)
+
+scala> df.select(to_csv($<span class="hljs-string">"_3"</span>)).show
++---------------+
+|structtocsv(_3)|
++---------------+
+| <span class="hljs-number">0</span>,<span class="hljs-number">3.9</span>,abc|
+| <span class="hljs-number">2</span>,<span class="hljs-number">0.4</span>,<span class="hljs-function"><span class="hljs-keyword">def</span><span class="hljs-title">|</span></span>
++---------------+
+</code></pre>
+<p><div id="page-footer"><hr><!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<p><sub><font color="gray">
+Apache Hivemall is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator.
+</font></sub></p>
+</div></p>
+
+
+ </section>
+
+ </div>
+ <div class="search-results">
+ <div class="has-results">
+
+ <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
+ <ul class="search-results-list"></ul>
+
+ </div>
+ <div class="no-results">
+
+ <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
+
+ </div>
+ </div>
+</div>
+
+ </div>
+ </div>
+
+ </div>
+
+
+
+
+ </div>
+
+ <script>
+ var gitbook = gitbook || [];
+ gitbook.push(function() {
+ gitbook.page.hasChanged({"page":{"title":"Other utility functions","level":"12.4.2","depth":2,"next":{"title":"Hivemall on Apache Spark","level":"13.1","depth":1,"url":"https://github.com/maropu/hivemall-spark","ref":"https://github.com/maropu/hivemall-spark","articles":[]},"previous":{"title":"Top-k Join processing","level":"12.4.1","depth":2,"path":"spark/misc/topk_join.md","ref":"spark/misc/topk_join.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/in
cubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"ancho
rjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"spark/misc/functions.md","mtime":"2017-03-24T08:36:30.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-04-24T10:44:53.699Z"},"basePath":"../..","book":{"language":""}});
+ });
+ </script>
+</div>
+
+
+ <script src="../../gitbook/gitbook.js"></script>
+ <script src="../../gitbook/theme.js"></script>
+
+
+ <script src="../../gitbook/gitbook-plugin-edit-link/plugin.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-github/plugin.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-splitter/splitter.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-etoc/plugin.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-toggle-chapters/toggle.js"></script>
+
+
+
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/3.1.1/anchor.min.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-anchorjs/anchor-style.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-search/search-engine.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-search/search.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-sharing/buttons.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
+
+
+
+ <script src="../../gitbook/gitbook-plugin-theme-api/theme-api.js"></script>
+
+
+
+ </body>
+</html>
+
http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/06c5398a/userguide/spark/misc/misc.html
----------------------------------------------------------------------
diff --git a/userguide/spark/misc/misc.html b/userguide/spark/misc/misc.html
index d4f39cd..2239b8d 100644
--- a/userguide/spark/misc/misc.html
+++ b/userguide/spark/misc/misc.html
@@ -1648,17 +1648,61 @@
- <li class="header">Part X - Hivemall on Spark</li>
+ <li class="header">Part X - Clustering</li>
- <li class="chapter " data-level="10.1" data-path="../getting_started/">
+ <li class="chapter " data-level="10.1" data-path="../../clustering/lda.html">
- <a href="../getting_started/">
+ <a href="../../clustering/lda.html">
<b>10.1.</b>
+ Latent Dirichlet Allocation
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part XI - GeoSpatial functions</li>
+
+
+
+ <li class="chapter " data-level="11.1" data-path="../../geospatial/latlon.html">
+
+ <a href="../../geospatial/latlon.html">
+
+
+ <b>11.1.</b>
+
+ Lat/Lon functions
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part XII - Hivemall on Spark</li>
+
+
+
+ <li class="chapter " data-level="12.1" data-path="../getting_started/">
+
+ <a href="../getting_started/">
+
+
+ <b>12.1.</b>
+
Getting Started
</a>
@@ -1668,12 +1712,12 @@
<ul class="articles">
- <li class="chapter " data-level="10.1.1" data-path="../getting_started/installation.html">
+ <li class="chapter " data-level="12.1.1" data-path="../getting_started/installation.html">
<a href="../getting_started/installation.html">
- <b>10.1.1.</b>
+ <b>12.1.1.</b>
Installation
@@ -1688,12 +1732,12 @@
</li>
- <li class="chapter " data-level="10.2" data-path="../binaryclass/">
+ <li class="chapter " data-level="12.2" data-path="../binaryclass/">
<a href="../binaryclass/">
- <b>10.2.</b>
+ <b>12.2.</b>
Binary Classification
@@ -1704,12 +1748,12 @@
<ul class="articles">
- <li class="chapter " data-level="10.2.1" data-path="../binaryclass/a9a_df.html">
+ <li class="chapter " data-level="12.2.1" data-path="../binaryclass/a9a_df.html">
<a href="../binaryclass/a9a_df.html">
- <b>10.2.1.</b>
+ <b>12.2.1.</b>
a9a Tutorial for DataFrame
@@ -1724,12 +1768,12 @@
</li>
- <li class="chapter " data-level="10.3" data-path="../binaryclass/">
+ <li class="chapter " data-level="12.3" data-path="../binaryclass/">
<a href="../binaryclass/">
- <b>10.3.</b>
+ <b>12.3.</b>
Regression
@@ -1740,12 +1784,12 @@
<ul class="articles">
- <li class="chapter " data-level="10.3.1" data-path="../regression/e2006_df.html">
+ <li class="chapter " data-level="12.3.1" data-path="../regression/e2006_df.html">
<a href="../regression/e2006_df.html">
- <b>10.3.1.</b>
+ <b>12.3.1.</b>
E2006-tfidf regression Tutorial for DataFrame
@@ -1760,12 +1804,12 @@
</li>
- <li class="chapter active" data-level="10.4" data-path="misc.html">
+ <li class="chapter active" data-level="12.4" data-path="misc.html">
<a href="misc.html">
- <b>10.4.</b>
+ <b>12.4.</b>
Generic features
@@ -1776,12 +1820,12 @@
<ul class="articles">
- <li class="chapter " data-level="10.4.1" data-path="topk_join.html">
+ <li class="chapter " data-level="12.4.1" data-path="topk_join.html">
<a href="topk_join.html">
- <b>10.4.1.</b>
+ <b>12.4.1.</b>
Top-k Join processing
@@ -1791,6 +1835,21 @@
</li>
+ <li class="chapter " data-level="12.4.2" data-path="functions.html">
+
+ <a href="functions.html">
+
+
+ <b>12.4.2.</b>
+
+ Other utility functions
+
+ </a>
+
+
+
+ </li>
+
</ul>
@@ -1799,16 +1858,16 @@
- <li class="header">Part X - External References</li>
+ <li class="header">Part XIII - External References</li>
- <li class="chapter " data-level="11.1" >
+ <li class="chapter " data-level="13.1" >
<a target="_blank" href="https://github.com/maropu/hivemall-spark">
- <b>11.1.</b>
+ <b>13.1.</b>
Hivemall on Apache Spark
@@ -1818,12 +1877,12 @@
</li>
- <li class="chapter " data-level="11.2" >
+ <li class="chapter " data-level="13.2" >
<a target="_blank" href="https://github.com/daijyc/hivemall/wiki/PigHome">
- <b>11.2.</b>
+ <b>13.2.</b>
Hivemall on Apache Pig
@@ -1951,7 +2010,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
- gitbook.page.hasChanged({"page":{"title":"Generic features","level":"10.4","depth":1,"next":{"title":"Top-k Join processing","level":"10.4.1","depth":2,"path":"spark/misc/topk_join.md","ref":"spark/misc/topk_join.md","articles":[]},"previous":{"title":"E2006-tfidf regression Tutorial for DataFrame","level":"10.3.1","depth":2,"path":"spark/regression/e2006_df.md","ref":"spark/regression/e2006_df.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-h
ivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"se
lector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"spark/misc/misc.md","mtime":"2017-02-05T01:22:37.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-03-02T16:12:56.998Z"},"basePath":"../..","book":{"language":""}});
+ gitbook.page.hasChanged({"page":{"title":"Generic features","level":"12.4","depth":1,"next":{"title":"Top-k Join processing","level":"12.4.1","depth":2,"path":"spark/misc/topk_join.md","ref":"spark/misc/topk_join.md","articles":[]},"previous":{"title":"E2006-tfidf regression Tutorial for DataFrame","level":"12.3.1","depth":2,"path":"spark/regression/e2006_df.md","ref":"spark/regression/e2006_df.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-h
ivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"se
lector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"spark/misc/misc.md","mtime":"2017-02-05T01:22:37.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-04-24T10:44:53.699Z"},"basePath":"../..","book":{"language":""}});
});
</script>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/06c5398a/userguide/spark/misc/topk_join.html
----------------------------------------------------------------------
diff --git a/userguide/spark/misc/topk_join.html b/userguide/spark/misc/topk_join.html
index bc1094e..f3334b5 100644
--- a/userguide/spark/misc/topk_join.html
+++ b/userguide/spark/misc/topk_join.html
@@ -97,6 +97,8 @@
<link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
+ <link rel="next" href="functions.html" />
+
<link rel="prev" href="misc.html" />
@@ -1646,17 +1648,61 @@
- <li class="header">Part X - Hivemall on Spark</li>
+ <li class="header">Part X - Clustering</li>
- <li class="chapter " data-level="10.1" data-path="../getting_started/">
+ <li class="chapter " data-level="10.1" data-path="../../clustering/lda.html">
- <a href="../getting_started/">
+ <a href="../../clustering/lda.html">
<b>10.1.</b>
+ Latent Dirichlet Allocation
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part XI - GeoSpatial functions</li>
+
+
+
+ <li class="chapter " data-level="11.1" data-path="../../geospatial/latlon.html">
+
+ <a href="../../geospatial/latlon.html">
+
+
+ <b>11.1.</b>
+
+ Lat/Lon functions
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part XII - Hivemall on Spark</li>
+
+
+
+ <li class="chapter " data-level="12.1" data-path="../getting_started/">
+
+ <a href="../getting_started/">
+
+
+ <b>12.1.</b>
+
Getting Started
</a>
@@ -1666,12 +1712,12 @@
<ul class="articles">
- <li class="chapter " data-level="10.1.1" data-path="../getting_started/installation.html">
+ <li class="chapter " data-level="12.1.1" data-path="../getting_started/installation.html">
<a href="../getting_started/installation.html">
- <b>10.1.1.</b>
+ <b>12.1.1.</b>
Installation
@@ -1686,12 +1732,12 @@
</li>
- <li class="chapter " data-level="10.2" data-path="../binaryclass/">
+ <li class="chapter " data-level="12.2" data-path="../binaryclass/">
<a href="../binaryclass/">
- <b>10.2.</b>
+ <b>12.2.</b>
Binary Classification
@@ -1702,12 +1748,12 @@
<ul class="articles">
- <li class="chapter " data-level="10.2.1" data-path="../binaryclass/a9a_df.html">
+ <li class="chapter " data-level="12.2.1" data-path="../binaryclass/a9a_df.html">
<a href="../binaryclass/a9a_df.html">
- <b>10.2.1.</b>
+ <b>12.2.1.</b>
a9a Tutorial for DataFrame
@@ -1722,12 +1768,12 @@
</li>
- <li class="chapter " data-level="10.3" data-path="../binaryclass/">
+ <li class="chapter " data-level="12.3" data-path="../binaryclass/">
<a href="../binaryclass/">
- <b>10.3.</b>
+ <b>12.3.</b>
Regression
@@ -1738,12 +1784,12 @@
<ul class="articles">
- <li class="chapter " data-level="10.3.1" data-path="../regression/e2006_df.html">
+ <li class="chapter " data-level="12.3.1" data-path="../regression/e2006_df.html">
<a href="../regression/e2006_df.html">
- <b>10.3.1.</b>
+ <b>12.3.1.</b>
E2006-tfidf regression Tutorial for DataFrame
@@ -1758,12 +1804,12 @@
</li>
- <li class="chapter " data-level="10.4" data-path="misc.html">
+ <li class="chapter " data-level="12.4" data-path="misc.html">
<a href="misc.html">
- <b>10.4.</b>
+ <b>12.4.</b>
Generic features
@@ -1774,12 +1820,12 @@
<ul class="articles">
- <li class="chapter active" data-level="10.4.1" data-path="topk_join.html">
+ <li class="chapter active" data-level="12.4.1" data-path="topk_join.html">
<a href="topk_join.html">
- <b>10.4.1.</b>
+ <b>12.4.1.</b>
Top-k Join processing
@@ -1789,6 +1835,21 @@
</li>
+ <li class="chapter " data-level="12.4.2" data-path="functions.html">
+
+ <a href="functions.html">
+
+
+ <b>12.4.2.</b>
+
+ Other utility functions
+
+ </a>
+
+
+
+ </li>
+
</ul>
@@ -1797,16 +1858,16 @@
- <li class="header">Part X - External References</li>
+ <li class="header">Part XIII - External References</li>
- <li class="chapter " data-level="11.1" >
+ <li class="chapter " data-level="13.1" >
<a target="_blank" href="https://github.com/maropu/hivemall-spark">
- <b>11.1.</b>
+ <b>13.1.</b>
Hivemall on Apache Spark
@@ -1816,12 +1877,12 @@
</li>
- <li class="chapter " data-level="11.2" >
+ <li class="chapter " data-level="13.2" >
<a target="_blank" href="https://github.com/daijyc/hivemall/wiki/PigHome">
- <b>11.2.</b>
+ <b>13.2.</b>
Hivemall on Apache Pig
@@ -2128,7 +2189,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
- gitbook.page.hasChanged({"page":{"title":"Top-k Join processing","level":"10.4.1","depth":2,"next":{"title":"Hivemall on Apache Spark","level":"11.1","depth":1,"url":"https://github.com/maropu/hivemall-spark","ref":"https://github.com/maropu/hivemall-spark","articles":[]},"previous":{"title":"Generic features","level":"10.4","depth":1,"path":"spark/misc/misc.md","ref":"spark/misc/misc.md","articles":[{"title":"Top-k Join processing","level":"10.4.1","depth":2,"path":"spark/misc/topk_join.md","ref":"spark/misc/topk_join.md","articles":[]}]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"e
mphasize":{},"callouts":{},"etoc":{"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"s
tyles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"spark/misc/topk_join.md","mtime":"2017-02-05T01:22:00.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-03-02T16:12:56.998Z"},"basePath":"../..","book":{"language":""}});
+ gitbook.page.hasChanged({"page":{"title":"Top-k Join processing","level":"12.4.1","depth":2,"next":{"title":"Other utility functions","level":"12.4.2","depth":2,"path":"spark/misc/functions.md","ref":"spark/misc/functions.md","articles":[]},"previous":{"title":"Generic features","level":"12.4","depth":1,"path":"spark/misc/misc.md","ref":"spark/misc/misc.md","articles":[{"title":"Top-k Join processing","level":"12.4.1","depth":2,"path":"spark/misc/topk_join.md","ref":"spark/misc/topk_join.md","articles":[]},{"title":"Other utility functions","level":"12.4.2","depth":2,"path":"spark/misc/functions.md","ref":"spark/misc/functions.md","articles":[]}]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles
/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hive
mall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"spark/misc/topk_join.md","mtime":"2017-02-05T01:22:00.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-04-24T10:44:53.699
Z"},"basePath":"../..","book":{"language":""}});
});
</script>
</div>