You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2017/05/08 09:01:08 UTC
[14/19] incubator-hivemall-site git commit: Added feature binning
documentation
http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/15b9991a/userguide/ft_engineering/feature_selection.html
----------------------------------------------------------------------
diff --git a/userguide/ft_engineering/feature_selection.html b/userguide/ft_engineering/feature_selection.html
deleted file mode 100644
index ab2effd..0000000
--- a/userguide/ft_engineering/feature_selection.html
+++ /dev/null
@@ -1,2329 +0,0 @@
-
-<!DOCTYPE HTML>
-<html lang="" >
- <head>
- <meta charset="UTF-8">
- <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
- <title>Feature selection ยท Hivemall User Manual</title>
- <meta http-equiv="X-UA-Compatible" content="IE=edge" />
- <meta name="description" content="">
- <meta name="generator" content="GitBook 3.2.2">
-
-
-
-
- <link rel="stylesheet" href="../gitbook/style.css">
-
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-splitter/splitter.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-etoc/plugin.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-callouts/plugin.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-toggle-chapters/toggle.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-codeblock-filename/block.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-multipart/multipart.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-katex/katex.min.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-emphasize/plugin.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
-
-
-
- <link rel="stylesheet" href="../gitbook/gitbook-plugin-theme-api/theme-api.css">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <meta name="HandheldFriendly" content="true"/>
- <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
- <meta name="apple-mobile-web-app-capable" content="yes">
- <meta name="apple-mobile-web-app-status-bar-style" content="black">
- <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
- <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
-
-
- <link rel="next" href="../eval/stat_eval.html" />
-
-
- <link rel="prev" href="quantify.html" />
-
-
- </head>
- <body>
-
-<div class="book">
- <div class="book-summary">
-
-
-<div id="book-search-input" role="search">
- <input type="text" placeholder="Type to search" />
-</div>
-
-
- <nav role="navigation">
-
-
-
-<ul class="summary">
-
-
-
-
- <li>
- <a href="http://hivemall.incubator.apache.org/" target="_blank" class="custom-link"><i class="fa fa-home"></i> Home</a>
- </li>
-
-
-
-
- <li class="divider"></li>
-
-
-
-
- <li class="header">TABLE OF CONTENTS</li>
-
-
-
- <li class="chapter " data-level="1.1" data-path="../">
-
- <a href="../">
-
-
- <b>1.1.</b>
-
- Introduction
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.2" data-path="../getting_started/">
-
- <a href="../getting_started/">
-
-
- <b>1.2.</b>
-
- Getting Started
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="1.2.1" data-path="../getting_started/installation.html">
-
- <a href="../getting_started/installation.html">
-
-
- <b>1.2.1.</b>
-
- Installation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.2.2" data-path="../getting_started/permanent-functions.html">
-
- <a href="../getting_started/permanent-functions.html">
-
-
- <b>1.2.2.</b>
-
- Install as permanent functions
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.2.3" data-path="../getting_started/input-format.html">
-
- <a href="../getting_started/input-format.html">
-
-
- <b>1.2.3.</b>
-
- Input Format
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="1.3" data-path="../tips/">
-
- <a href="../tips/">
-
-
- <b>1.3.</b>
-
- Tips for Effective Hivemall
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="1.3.1" data-path="../tips/addbias.html">
-
- <a href="../tips/addbias.html">
-
-
- <b>1.3.1.</b>
-
- Explicit addBias() for better prediction
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.3.2" data-path="../tips/rand_amplify.html">
-
- <a href="../tips/rand_amplify.html">
-
-
- <b>1.3.2.</b>
-
- Use rand_amplify() to better prediction results
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.3.3" data-path="../tips/rt_prediction.html">
-
- <a href="../tips/rt_prediction.html">
-
-
- <b>1.3.3.</b>
-
- Real-time Prediction on RDBMS
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.3.4" data-path="../tips/ensemble_learning.html">
-
- <a href="../tips/ensemble_learning.html">
-
-
- <b>1.3.4.</b>
-
- Ensemble learning for stable prediction
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.3.5" data-path="../tips/mixserver.html">
-
- <a href="../tips/mixserver.html">
-
-
- <b>1.3.5.</b>
-
- Mixing models for a better prediction convergence (MIX server)
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.3.6" data-path="../tips/emr.html">
-
- <a href="../tips/emr.html">
-
-
- <b>1.3.6.</b>
-
- Run Hivemall on Amazon Elastic MapReduce
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="1.4" data-path="../tips/general_tips.html">
-
- <a href="../tips/general_tips.html">
-
-
- <b>1.4.</b>
-
- General Hive/Hadoop tips
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="1.4.1" data-path="../tips/rowid.html">
-
- <a href="../tips/rowid.html">
-
-
- <b>1.4.1.</b>
-
- Adding rowid for each row
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.4.2" data-path="../tips/hadoop_tuning.html">
-
- <a href="../tips/hadoop_tuning.html">
-
-
- <b>1.4.2.</b>
-
- Hadoop tuning for Hivemall
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="1.5" data-path="../troubleshooting/">
-
- <a href="../troubleshooting/">
-
-
- <b>1.5.</b>
-
- Troubleshooting
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="1.5.1" data-path="../troubleshooting/oom.html">
-
- <a href="../troubleshooting/oom.html">
-
-
- <b>1.5.1.</b>
-
- OutOfMemoryError in training
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.5.2" data-path="../troubleshooting/mapjoin_task_error.html">
-
- <a href="../troubleshooting/mapjoin_task_error.html">
-
-
- <b>1.5.2.</b>
-
- SemanticException Generate Map Join Task Error: Cannot serialize object
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.5.3" data-path="../troubleshooting/asterisk.html">
-
- <a href="../troubleshooting/asterisk.html">
-
-
- <b>1.5.3.</b>
-
- Asterisk argument for UDTF does not work
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.5.4" data-path="../troubleshooting/num_mappers.html">
-
- <a href="../troubleshooting/num_mappers.html">
-
-
- <b>1.5.4.</b>
-
- The number of mappers is less than input splits in Hadoop 2.x
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="1.5.5" data-path="../troubleshooting/mapjoin_classcastex.html">
-
- <a href="../troubleshooting/mapjoin_classcastex.html">
-
-
- <b>1.5.5.</b>
-
- Map-side Join causes ClassCastException on Tez
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
-
-
-
- <li class="header">Part II - Generic Features</li>
-
-
-
- <li class="chapter " data-level="2.1" data-path="../misc/generic_funcs.html">
-
- <a href="../misc/generic_funcs.html">
-
-
- <b>2.1.</b>
-
- List of generic Hivemall functions
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="2.2" data-path="../misc/topk.html">
-
- <a href="../misc/topk.html">
-
-
- <b>2.2.</b>
-
- Efficient Top-K query processing
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="2.3" data-path="../misc/tokenizer.html">
-
- <a href="../misc/tokenizer.html">
-
-
- <b>2.3.</b>
-
- English/Japanese Text Tokenizer
-
- </a>
-
-
-
- </li>
-
-
-
-
- <li class="header">Part III - Feature Engineering</li>
-
-
-
- <li class="chapter " data-level="3.1" data-path="scaling.html">
-
- <a href="scaling.html">
-
-
- <b>3.1.</b>
-
- Feature Scaling
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="3.2" data-path="hashing.html">
-
- <a href="hashing.html">
-
-
- <b>3.2.</b>
-
- Feature Hashing
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="3.3" data-path="tfidf.html">
-
- <a href="tfidf.html">
-
-
- <b>3.3.</b>
-
- TF-IDF calculation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="3.4" data-path="ft_trans.html">
-
- <a href="ft_trans.html">
-
-
- <b>3.4.</b>
-
- FEATURE TRANSFORMATION
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="3.4.1" data-path="vectorizer.html">
-
- <a href="vectorizer.html">
-
-
- <b>3.4.1.</b>
-
- Vectorize Features
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="3.4.2" data-path="quantify.html">
-
- <a href="quantify.html">
-
-
- <b>3.4.2.</b>
-
- Quantify non-number features
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter active" data-level="3.5" data-path="feature_selection.html">
-
- <a href="feature_selection.html">
-
-
- <b>3.5.</b>
-
- Feature selection
-
- </a>
-
-
-
- </li>
-
-
-
-
- <li class="header">Part IV - Evaluation</li>
-
-
-
- <li class="chapter " data-level="4.1" data-path="../eval/stat_eval.html">
-
- <a href="../eval/stat_eval.html">
-
-
- <b>4.1.</b>
-
- Statistical evaluation of a prediction model
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="4.1.1" data-path="../eval/auc.html">
-
- <a href="../eval/auc.html">
-
-
- <b>4.1.1.</b>
-
- Area Under the ROC Curve
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="4.2" data-path="../eval/rank.html">
-
- <a href="../eval/rank.html">
-
-
- <b>4.2.</b>
-
- Ranking Measures
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="4.3" data-path="../eval/datagen.html">
-
- <a href="../eval/datagen.html">
-
-
- <b>4.3.</b>
-
- Data Generation
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="4.3.1" data-path="../eval/lr_datagen.html">
-
- <a href="../eval/lr_datagen.html">
-
-
- <b>4.3.1.</b>
-
- Logistic Regression data generation
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
-
-
-
- <li class="header">Part V - Binary classification</li>
-
-
-
- <li class="chapter " data-level="5.1" data-path="../binaryclass/a9a.html">
-
- <a href="../binaryclass/a9a.html">
-
-
- <b>5.1.</b>
-
- a9a Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="5.1.1" data-path="../binaryclass/a9a_dataset.html">
-
- <a href="../binaryclass/a9a_dataset.html">
-
-
- <b>5.1.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="5.1.2" data-path="../binaryclass/a9a_lr.html">
-
- <a href="../binaryclass/a9a_lr.html">
-
-
- <b>5.1.2.</b>
-
- Logistic Regression
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="5.1.3" data-path="../binaryclass/a9a_minibatch.html">
-
- <a href="../binaryclass/a9a_minibatch.html">
-
-
- <b>5.1.3.</b>
-
- Mini-batch Gradient Descent
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="5.2" data-path="../binaryclass/news20.html">
-
- <a href="../binaryclass/news20.html">
-
-
- <b>5.2.</b>
-
- News20 Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="5.2.1" data-path="../binaryclass/news20_dataset.html">
-
- <a href="../binaryclass/news20_dataset.html">
-
-
- <b>5.2.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="5.2.2" data-path="../binaryclass/news20_pa.html">
-
- <a href="../binaryclass/news20_pa.html">
-
-
- <b>5.2.2.</b>
-
- Perceptron, Passive Aggressive
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="5.2.3" data-path="../binaryclass/news20_scw.html">
-
- <a href="../binaryclass/news20_scw.html">
-
-
- <b>5.2.3.</b>
-
- CW, AROW, SCW
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="5.2.4" data-path="../binaryclass/news20_adagrad.html">
-
- <a href="../binaryclass/news20_adagrad.html">
-
-
- <b>5.2.4.</b>
-
- AdaGradRDA, AdaGrad, AdaDelta
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="5.3" data-path="../binaryclass/kdd2010a.html">
-
- <a href="../binaryclass/kdd2010a.html">
-
-
- <b>5.3.</b>
-
- KDD2010a Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="5.3.1" data-path="../binaryclass/kdd2010a_dataset.html">
-
- <a href="../binaryclass/kdd2010a_dataset.html">
-
-
- <b>5.3.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="5.3.2" data-path="../binaryclass/kdd2010a_scw.html">
-
- <a href="../binaryclass/kdd2010a_scw.html">
-
-
- <b>5.3.2.</b>
-
- PA, CW, AROW, SCW
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="5.4" data-path="../binaryclass/kdd2010b.html">
-
- <a href="../binaryclass/kdd2010b.html">
-
-
- <b>5.4.</b>
-
- KDD2010b Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="5.4.1" data-path="../binaryclass/kdd2010b_dataset.html">
-
- <a href="../binaryclass/kdd2010b_dataset.html">
-
-
- <b>5.4.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="5.4.2" data-path="../binaryclass/kdd2010b_arow.html">
-
- <a href="../binaryclass/kdd2010b_arow.html">
-
-
- <b>5.4.2.</b>
-
- AROW
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="5.5" data-path="../binaryclass/webspam.html">
-
- <a href="../binaryclass/webspam.html">
-
-
- <b>5.5.</b>
-
- Webspam Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="5.5.1" data-path="../binaryclass/webspam_dataset.html">
-
- <a href="../binaryclass/webspam_dataset.html">
-
-
- <b>5.5.1.</b>
-
- Data pareparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="5.5.2" data-path="../binaryclass/webspam_scw.html">
-
- <a href="../binaryclass/webspam_scw.html">
-
-
- <b>5.5.2.</b>
-
- PA1, AROW, SCW
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="5.6" data-path="../binaryclass/titanic_rf.html">
-
- <a href="../binaryclass/titanic_rf.html">
-
-
- <b>5.6.</b>
-
- Kaggle Titanic Tutorial
-
- </a>
-
-
-
- </li>
-
-
-
-
- <li class="header">Part VI - Multiclass classification</li>
-
-
-
- <li class="chapter " data-level="6.1" data-path="../multiclass/news20.html">
-
- <a href="../multiclass/news20.html">
-
-
- <b>6.1.</b>
-
- News20 Multiclass Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="6.1.1" data-path="../multiclass/news20_dataset.html">
-
- <a href="../multiclass/news20_dataset.html">
-
-
- <b>6.1.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="6.1.2" data-path="../multiclass/news20_one-vs-the-rest_dataset.html">
-
- <a href="../multiclass/news20_one-vs-the-rest_dataset.html">
-
-
- <b>6.1.2.</b>
-
- Data preparation for one-vs-the-rest classifiers
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="6.1.3" data-path="../multiclass/news20_pa.html">
-
- <a href="../multiclass/news20_pa.html">
-
-
- <b>6.1.3.</b>
-
- PA
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="6.1.4" data-path="../multiclass/news20_scw.html">
-
- <a href="../multiclass/news20_scw.html">
-
-
- <b>6.1.4.</b>
-
- CW, AROW, SCW
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="6.1.5" data-path="../multiclass/news20_ensemble.html">
-
- <a href="../multiclass/news20_ensemble.html">
-
-
- <b>6.1.5.</b>
-
- Ensemble learning
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="6.1.6" data-path="../multiclass/news20_one-vs-the-rest.html">
-
- <a href="../multiclass/news20_one-vs-the-rest.html">
-
-
- <b>6.1.6.</b>
-
- one-vs-the-rest classifier
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="6.2" data-path="../multiclass/iris.html">
-
- <a href="../multiclass/iris.html">
-
-
- <b>6.2.</b>
-
- Iris Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="6.2.1" data-path="../multiclass/iris_dataset.html">
-
- <a href="../multiclass/iris_dataset.html">
-
-
- <b>6.2.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="6.2.2" data-path="../multiclass/iris_scw.html">
-
- <a href="../multiclass/iris_scw.html">
-
-
- <b>6.2.2.</b>
-
- SCW
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="6.2.3" data-path="../multiclass/iris_randomforest.html">
-
- <a href="../multiclass/iris_randomforest.html">
-
-
- <b>6.2.3.</b>
-
- RandomForest
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
-
-
-
- <li class="header">Part VII - Regression</li>
-
-
-
- <li class="chapter " data-level="7.1" data-path="../regression/e2006.html">
-
- <a href="../regression/e2006.html">
-
-
- <b>7.1.</b>
-
- E2006-tfidf regression Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="7.1.1" data-path="../regression/e2006_dataset.html">
-
- <a href="../regression/e2006_dataset.html">
-
-
- <b>7.1.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="7.1.2" data-path="../regression/e2006_arow.html">
-
- <a href="../regression/e2006_arow.html">
-
-
- <b>7.1.2.</b>
-
- Passive Aggressive, AROW
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="7.2" data-path="../regression/kddcup12tr2.html">
-
- <a href="../regression/kddcup12tr2.html">
-
-
- <b>7.2.</b>
-
- KDDCup 2012 track 2 CTR prediction Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="7.2.1" data-path="../regression/kddcup12tr2_dataset.html">
-
- <a href="../regression/kddcup12tr2_dataset.html">
-
-
- <b>7.2.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="7.2.2" data-path="../regression/kddcup12tr2_lr.html">
-
- <a href="../regression/kddcup12tr2_lr.html">
-
-
- <b>7.2.2.</b>
-
- Logistic Regression, Passive Aggressive
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="7.2.3" data-path="../regression/kddcup12tr2_lr_amplify.html">
-
- <a href="../regression/kddcup12tr2_lr_amplify.html">
-
-
- <b>7.2.3.</b>
-
- Logistic Regression with Amplifier
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="7.2.4" data-path="../regression/kddcup12tr2_adagrad.html">
-
- <a href="../regression/kddcup12tr2_adagrad.html">
-
-
- <b>7.2.4.</b>
-
- AdaGrad, AdaDelta
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
-
-
-
- <li class="header">Part VIII - Recommendation</li>
-
-
-
- <li class="chapter " data-level="8.1" data-path="../recommend/cf.html">
-
- <a href="../recommend/cf.html">
-
-
- <b>8.1.</b>
-
- Collaborative Filtering
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="8.1.1" data-path="../recommend/item_based_cf.html">
-
- <a href="../recommend/item_based_cf.html">
-
-
- <b>8.1.1.</b>
-
- Item-based Collaborative Filtering
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="8.2" data-path="../recommend/news20.html">
-
- <a href="../recommend/news20.html">
-
-
- <b>8.2.</b>
-
- News20 related article recommendation Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="8.2.1" data-path="../multiclass/news20_dataset.html">
-
- <a href="../multiclass/news20_dataset.html">
-
-
- <b>8.2.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="8.2.2" data-path="../recommend/news20_jaccard.html">
-
- <a href="../recommend/news20_jaccard.html">
-
-
- <b>8.2.2.</b>
-
- LSH/Minhash and Jaccard Similarity
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="8.2.3" data-path="../recommend/news20_knn.html">
-
- <a href="../recommend/news20_knn.html">
-
-
- <b>8.2.3.</b>
-
- LSH/Minhash and Brute-Force Search
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="8.2.4" data-path="../recommend/news20_bbit_minhash.html">
-
- <a href="../recommend/news20_bbit_minhash.html">
-
-
- <b>8.2.4.</b>
-
- kNN search using b-Bits Minhash
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="8.3" data-path="../recommend/movielens.html">
-
- <a href="../recommend/movielens.html">
-
-
- <b>8.3.</b>
-
- MovieLens movie recommendation Tutorial
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="8.3.1" data-path="../recommend/movielens_dataset.html">
-
- <a href="../recommend/movielens_dataset.html">
-
-
- <b>8.3.1.</b>
-
- Data preparation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="8.3.2" data-path="../recommend/movielens_mf.html">
-
- <a href="../recommend/movielens_mf.html">
-
-
- <b>8.3.2.</b>
-
- Matrix Factorization
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="8.3.3" data-path="../recommend/movielens_fm.html">
-
- <a href="../recommend/movielens_fm.html">
-
-
- <b>8.3.3.</b>
-
- Factorization Machine
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="8.3.4" data-path="../recommend/movielens_cv.html">
-
- <a href="../recommend/movielens_cv.html">
-
-
- <b>8.3.4.</b>
-
- 10-fold Cross Validation (Matrix Factorization)
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
-
-
-
- <li class="header">Part IX - Anomaly Detection</li>
-
-
-
- <li class="chapter " data-level="9.1" data-path="../anomaly/lof.html">
-
- <a href="../anomaly/lof.html">
-
-
- <b>9.1.</b>
-
- Outlier Detection using Local Outlier Factor (LOF)
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="9.2" data-path="../anomaly/sst.html">
-
- <a href="../anomaly/sst.html">
-
-
- <b>9.2.</b>
-
- Change-Point Detection using Singular Spectrum Transformation (SST)
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="9.3" data-path="../anomaly/changefinder.html">
-
- <a href="../anomaly/changefinder.html">
-
-
- <b>9.3.</b>
-
- ChangeFinder: Detecting Outlier and Change-Point Simultaneously
-
- </a>
-
-
-
- </li>
-
-
-
-
- <li class="header">Part X - Clustering</li>
-
-
-
- <li class="chapter " data-level="10.1" data-path="../clustering/lda.html">
-
- <a href="../clustering/lda.html">
-
-
- <b>10.1.</b>
-
- Latent Dirichlet Allocation
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="10.2" data-path="../clustering/plsa.html">
-
- <a href="../clustering/plsa.html">
-
-
- <b>10.2.</b>
-
- Probabilistic Latent Semantic Analysis
-
- </a>
-
-
-
- </li>
-
-
-
-
- <li class="header">Part XI - GeoSpatial functions</li>
-
-
-
- <li class="chapter " data-level="11.1" data-path="../geospatial/latlon.html">
-
- <a href="../geospatial/latlon.html">
-
-
- <b>11.1.</b>
-
- Lat/Lon functions
-
- </a>
-
-
-
- </li>
-
-
-
-
- <li class="header">Part XII - Hivemall on Spark</li>
-
-
-
- <li class="chapter " data-level="12.1" data-path="../spark/getting_started/">
-
- <a href="../spark/getting_started/">
-
-
- <b>12.1.</b>
-
- Getting Started
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="12.1.1" data-path="../spark/getting_started/installation.html">
-
- <a href="../spark/getting_started/installation.html">
-
-
- <b>12.1.1.</b>
-
- Installation
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="12.2" data-path="../spark/binaryclass/">
-
- <a href="../spark/binaryclass/">
-
-
- <b>12.2.</b>
-
- Binary Classification
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="12.2.1" data-path="../spark/binaryclass/a9a_df.html">
-
- <a href="../spark/binaryclass/a9a_df.html">
-
-
- <b>12.2.1.</b>
-
- a9a Tutorial for DataFrame
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="12.3" data-path="../spark/binaryclass/">
-
- <a href="../spark/binaryclass/">
-
-
- <b>12.3.</b>
-
- Regression
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="12.3.1" data-path="../spark/regression/e2006_df.html">
-
- <a href="../spark/regression/e2006_df.html">
-
-
- <b>12.3.1.</b>
-
- E2006-tfidf regression Tutorial for DataFrame
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
- <li class="chapter " data-level="12.4" data-path="../spark/misc/misc.html">
-
- <a href="../spark/misc/misc.html">
-
-
- <b>12.4.</b>
-
- Generic features
-
- </a>
-
-
-
- <ul class="articles">
-
-
- <li class="chapter " data-level="12.4.1" data-path="../spark/misc/topk_join.html">
-
- <a href="../spark/misc/topk_join.html">
-
-
- <b>12.4.1.</b>
-
- Top-k Join processing
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="12.4.2" data-path="../spark/misc/functions.html">
-
- <a href="../spark/misc/functions.html">
-
-
- <b>12.4.2.</b>
-
- Other utility functions
-
- </a>
-
-
-
- </li>
-
-
- </ul>
-
- </li>
-
-
-
-
- <li class="header">Part XIII - Hivemall on Docker</li>
-
-
-
- <li class="chapter " data-level="13.1" data-path="../docker/getting_started.html">
-
- <a href="../docker/getting_started.html">
-
-
- <b>13.1.</b>
-
- Getting Started
-
- </a>
-
-
-
- </li>
-
-
-
-
- <li class="header">Part XIV - External References</li>
-
-
-
- <li class="chapter " data-level="14.1" >
-
- <a target="_blank" href="https://github.com/maropu/hivemall-spark">
-
-
- <b>14.1.</b>
-
- Hivemall on Apache Spark
-
- </a>
-
-
-
- </li>
-
- <li class="chapter " data-level="14.2" >
-
- <a target="_blank" href="https://github.com/daijyc/hivemall/wiki/PigHome">
-
-
- <b>14.2.</b>
-
- Hivemall on Apache Pig
-
- </a>
-
-
-
- </li>
-
-
-
-
- <li class="divider"></li>
-
- <li>
- <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
- Published with GitBook
- </a>
- </li>
-</ul>
-
-
- </nav>
-
-
- </div>
-
- <div class="book-body">
-
- <div class="body-inner">
-
-
-
-<div class="book-header" role="navigation">
-
-
- <!-- Title -->
- <h1>
- <i class="fa fa-circle-o-notch fa-spin"></i>
- <a href=".." >Feature selection</a>
- </h1>
-</div>
-
-
-
-
- <div class="page-wrapper" tabindex="-1" role="main">
- <div class="page-inner">
-
-<div id="book-search-results">
- <div class="search-noresults">
-
- <section class="normal markdown-section">
-
- <!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<p><a href="https://en.wikipedia.org/wiki/Feature_selection" target="_blank">Feature Selection</a> is the process of selecting a subset of relevant features for use in model construction. </p>
-<p>It is a useful technique to 1) improve prediction results by omitting redundant features, 2) to shorten training time, and 3) to know important features for prediction.</p>
-<p><em>Note: This feature is supported from Hivemall v0.5-rc.1 or later.</em></p>
-<!-- toc --><div id="toc" class="toc">
-
-<ul>
-<li><a href="#supported-feature-selection-algorithms">Supported Feature Selection algorithms</a></li>
-<li><a href="#usage">Usage</a><ul>
-<li><a href="#feature-selection-based-on-chi-square-test">Feature Selection based on Chi-square test</a></li>
-<li><a href="#feature-selection-based-on-signal-noise-ratio-snr">Feature Selection based on Signal Noise Ratio (SNR)</a></li>
-</ul>
-</li>
-<li><a href="#function-signatures">Function signatures</a><ul>
-<li><a href="#udaf-transposeanddotxarraynumber-yarraynumberarrayarraydouble">[UDAF] <code>transpose_and_dot(X::array<number>, Y::array<number>)::array<array<double>></code></a></li>
-<li><a href="#udf-selectkbestxarraynumber-importancelistarraynumber-kintarraydouble">[UDF] <code>select_k_best(X::array<number>, importance_list::array<number>, k::int)::array<double></code></a></li>
-<li><a href="#udf-chi2observedarrayarraynumber-expectedarrayarraynumberstructarraydouble-arraydouble">[UDF] <code>chi2(observed::array<array<number>>, expected::array<array<number>>)::struct<array<double>, array<double>></code></a></li>
-<li><a href="#udaf-snrxarraynumber-yarrayintarraydouble">[UDAF] <code>snr(X::array<number>, Y::array<int>)::array<double></code></a></li>
-</ul>
-</li>
-</ul>
-
-</div><!-- tocstop -->
-<h1 id="supported-feature-selection-algorithms">Supported Feature Selection algorithms</h1>
-<ul>
-<li>Chi-square (Chi2)<ul>
-<li>In statistics, the <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msup><mi>χ</mi><mn>2</mn></msup></mrow><annotation encoding="application/x-tex">\chi^2</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.8141079999999999em;"></span><span class="strut bottom" style="height:1.008548em;vertical-align:-0.19444em;"></span><span class="base textstyle uncramped"><span class="mord"><span class="mord mathit">χ</span><span class="vlist"><span style="top:-0.363em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle uncramped"><span class="mord mathrm">2</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span></span></span></span> test is applied to test the indep
endence of two even events. Chi-square statistics between every feature variable and the target variable can be applied to Feature Selection. Refer <a href="http://nlp.stanford.edu/IR-book/html/htmledition/feature-selectionchi2-feature-selection-1.html" target="_blank">this article</a> for Mathematical details.</li>
-</ul>
-</li>
-<li>Signal Noise Ratio (SNR)<ul>
-<li>The Signal Noise Ratio (SNR) is a univariate feature ranking metric, which can be used as a feature selection criterion for binary classification problems. SNR is defined as <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mi mathvariant="normal">∣</mi><msub><mi>μ</mi><mrow><mn>1</mn></mrow></msub><mo>−</mo><msub><mi>μ</mi><mrow><mn>2</mn></mrow></msub><mi mathvariant="normal">∣</mi><mi mathvariant="normal">/</mi><mo>(</mo><msub><mi>σ</mi><mrow><mn>1</mn></mrow></msub><mo>+</mo><msub><mi>σ</mi><mrow><mn>2</mn></mrow></msub><mo>)</mo></mrow><annotation encoding="application/x-tex">|\mu_{1} - \mu_{2}| / (\sigma_{1} + \sigma_{2})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom" style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord mathrm">∣</span><s
pan class="mord"><span class="mord mathit">μ</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:0em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord scriptstyle cramped"><span class="mord mathrm">1</span></span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mbin">−</span><span class="mord"><span class="mord mathit">μ</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:0em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord scriptstyle cramped"><span class="mord mathrm">2</span></span></span></span><span class="baseline-fix"><span class=
"fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mord mathrm">∣</span><span class="mord mathrm">/</span><span class="mopen">(</span><span class="mord"><span class="mord mathit" style="margin-right:0.03588em;">σ</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.03588em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord scriptstyle cramped"><span class="mord mathrm">1</span></span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mbin">+</span><span class="mord"><span class="mord mathit" style="margin-right:0.03588em;">σ</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.
03588em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord scriptstyle cramped"><span class="mord mathrm">2</span></span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mclose">)</span></span></span></span>, where <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msub><mi>μ</mi><mrow><mi>k</mi></mrow></msub></mrow><annotation encoding="application/x-tex">\mu_{k}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.43056em;"></span><span class="strut bottom" style="height:0.625em;vertical-align:-0.19444em;"></span><span class="base textstyle uncramped"><span class="mord"><span class="mord mathit">μ</span><span class="vlist"><span style="t
op:0.15em;margin-right:0.05em;margin-left:0em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord scriptstyle cramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span></span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span></span></span></span> is the mean value of the variable in classes <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mi>k</mi></mrow><annotation encoding="application/x-tex">k</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.69444em;"></span><span class="strut bottom" style="height:0.69444em;vertical-align:0em;"></span><span class="base textstyle uncramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span><
/span></span></span>, and <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msub><mi>σ</mi><mrow><mi>k</mi></mrow></msub></mrow><annotation encoding="application/x-tex">\sigma_{k}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.43056em;"></span><span class="strut bottom" style="height:0.58056em;vertical-align:-0.15em;"></span><span class="base textstyle uncramped"><span class="mord"><span class="mord mathit" style="margin-right:0.03588em;">σ</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.03588em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord scriptstyle cramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span></span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><
span style="font-size:0em;">​</span></span>​</span></span></span></span></span></span> is the standard deviations of the variable in classes <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mi>k</mi></mrow><annotation encoding="application/x-tex">k</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.69444em;"></span><span class="strut bottom" style="height:0.69444em;vertical-align:0em;"></span><span class="base textstyle uncramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span></span></span></span>. Clearly, features with larger SNR are useful for classification.</li>
-</ul>
-</li>
-</ul>
-<h1 id="usage">Usage</h1>
-<h2 id="feature-selection-based-on-chi-square-test">Feature Selection based on Chi-square test</h2>
-<pre><code class="lang-sql"><span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">TABLE</span> <span class="hljs-keyword">input</span> (
- X <span class="hljs-built_in">array</span><<span class="hljs-keyword">double</span>>, <span class="hljs-comment">-- features</span>
- Y <span class="hljs-built_in">array</span><<span class="hljs-built_in">int</span>> <span class="hljs-comment">-- binarized label</span>
-);
-
-<span class="hljs-keyword">set</span> hivevar:k=<span class="hljs-number">2</span>;
-
-WITH stats AS (
- <span class="hljs-keyword">SELECT</span>
- transpose_and_dot(Y, X) <span class="hljs-keyword">AS</span> observed, <span class="hljs-comment">-- array<array<double>>, shape = (n_classes, n_features)</span>
- array_sum(X) <span class="hljs-keyword">AS</span> feature_count, <span class="hljs-comment">-- n_features col vector, shape = (1, array<double>)</span>
- array_avg(Y) <span class="hljs-keyword">AS</span> class_prob <span class="hljs-comment">-- n_class col vector, shape = (1, array<double>)</span>
- <span class="hljs-keyword">FROM</span>
- <span class="hljs-keyword">input</span>
-),
-<span class="hljs-keyword">test</span> <span class="hljs-keyword">AS</span> (
- <span class="hljs-keyword">SELECT</span>
- transpose_and_dot(class_prob, feature_count) <span class="hljs-keyword">AS</span> expected <span class="hljs-comment">-- array<array<double>>, shape = (n_class, n_features)</span>
- <span class="hljs-keyword">FROM</span>
- stats
-),
-chi2 <span class="hljs-keyword">AS</span> (
- <span class="hljs-keyword">SELECT</span>
- chi2(r.observed, l.expected) <span class="hljs-keyword">AS</span> v <span class="hljs-comment">-- struct<array<double>, array<double>>, each shape = (1, n_features)</span>
- <span class="hljs-keyword">FROM</span>
- <span class="hljs-keyword">test</span> l
- <span class="hljs-keyword">CROSS</span> <span class="hljs-keyword">JOIN</span> stats r
-)
-<span class="hljs-keyword">SELECT</span>
- select_k_best(l.X, r.v.chi2, ${k}) <span class="hljs-keyword">as</span> features <span class="hljs-comment">-- top-k feature selection based on chi2 score</span>
-<span class="hljs-keyword">FROM</span>
- <span class="hljs-keyword">input</span> l
- <span class="hljs-keyword">CROSS</span> <span class="hljs-keyword">JOIN</span> chi2 r;
-</code></pre>
-<h2 id="feature-selection-based-on-signal-noise-ratio-snr">Feature Selection based on Signal Noise Ratio (SNR)</h2>
-<pre><code class="lang-sql"><span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">TABLE</span> <span class="hljs-keyword">input</span> (
- X <span class="hljs-built_in">array</span><<span class="hljs-keyword">double</span>>, <span class="hljs-comment">-- features</span>
- Y <span class="hljs-built_in">array</span><<span class="hljs-built_in">int</span>> <span class="hljs-comment">-- binarized label</span>
-);
-
-<span class="hljs-keyword">set</span> hivevar:k=<span class="hljs-number">2</span>;
-
-WITH snr AS (
- <span class="hljs-keyword">SELECT</span> snr(X, Y) <span class="hljs-keyword">AS</span> snr <span class="hljs-comment">-- aggregated SNR as array<double>, shape = (1, #features)</span>
- <span class="hljs-keyword">FROM</span> <span class="hljs-keyword">input</span>
-)
-<span class="hljs-keyword">SELECT</span>
- select_k_best(X, snr, ${k}) <span class="hljs-keyword">as</span> features
-<span class="hljs-keyword">FROM</span>
- <span class="hljs-keyword">input</span>
- <span class="hljs-keyword">CROSS</span> <span class="hljs-keyword">JOIN</span> snr;
-</code></pre>
-<h1 id="function-signatures">Function signatures</h1>
-<h3 id="udaf-transposeanddotxarraynumber-yarraynumberarrayarraydouble">[UDAF] <code>transpose_and_dot(X::array<number>, Y::array<number>)::array<array<double>></code></h3>
-<h5 id="input">Input</h5>
-<table>
-<thead>
-<tr>
-<th style="text-align:center"><code>array<number></code> X</th>
-<th style="text-align:center"><code>array<number></code> Y</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align:center">a row of matrix</td>
-<td style="text-align:center">a row of matrix</td>
-</tr>
-</tbody>
-</table>
-<h5 id="output">Output</h5>
-<table>
-<thead>
-<tr>
-<th style="text-align:center"><code>array<array<double>></code> dot product</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align:center"><code>dot(X.T, Y)</code> of shape = (X.#cols, Y.#cols)</td>
-</tr>
-</tbody>
-</table>
-<h3 id="udf-selectkbestxarraynumber-importancelistarraynumber-kintarraydouble">[UDF] <code>select_k_best(X::array<number>, importance_list::array<number>, k::int)::array<double></code></h3>
-<h5 id="input">Input</h5>
-<table>
-<thead>
-<tr>
-<th style="text-align:center"><code>array<number></code> X</th>
-<th style="text-align:center"><code>array<number></code> importance_list</th>
-<th style="text-align:center"><code>int</code> k</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align:center">feature vector</td>
-<td style="text-align:center">importance of each feature</td>
-<td style="text-align:center">the number of features to be selected</td>
-</tr>
-</tbody>
-</table>
-<h5 id="output">Output</h5>
-<table>
-<thead>
-<tr>
-<th style="text-align:center"><code>array<array<double>></code> k-best features</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align:center">top-k elements from feature vector <code>X</code> based on importance list</td>
-</tr>
-</tbody>
-</table>
-<h3 id="udf-chi2observedarrayarraynumber-expectedarrayarraynumberstructarraydouble-arraydouble">[UDF] <code>chi2(observed::array<array<number>>, expected::array<array<number>>)::struct<array<double>, array<double>></code></h3>
-<h5 id="input">Input</h5>
-<table>
-<thead>
-<tr>
-<th style="text-align:center"><code>array<number></code> observed</th>
-<th style="text-align:center"><code>array<number></code> expected</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align:center">observed features</td>
-<td style="text-align:center">expected features <code>dot(class_prob.T, feature_count)</code></td>
-</tr>
-</tbody>
-</table>
-<p>Both of <code>observed</code> and <code>expected</code> have a shape <code>(#classes, #features)</code></p>
-<h5 id="output">Output</h5>
-<table>
-<thead>
-<tr>
-<th style="text-align:center"><code>struct<array<double>, array<double>></code> importance_list</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align:center">chi2-value and p-value for each feature</td>
-</tr>
-</tbody>
-</table>
-<h3 id="udaf-snrxarraynumber-yarrayintarraydouble">[UDAF] <code>snr(X::array<number>, Y::array<int>)::array<double></code></h3>
-<h5 id="input">Input</h5>
-<table>
-<thead>
-<tr>
-<th style="text-align:center"><code>array<number></code> X</th>
-<th style="text-align:center"><code>array<int></code> Y</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align:center">feature vector</td>
-<td style="text-align:center">one hot label</td>
-</tr>
-</tbody>
-</table>
-<h5 id="output">Output</h5>
-<table>
-<thead>
-<tr>
-<th style="text-align:center"><code>array<double></code> importance_list</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align:center">Signal Noise Ratio for each feature</td>
-</tr>
-</tbody>
-</table>
-<p><div id="page-footer"><hr><!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<p><sub><font color="gray">
-Apache Hivemall is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator.
-</font></sub></p>
-</div></p>
-
-
- </section>
-
- </div>
- <div class="search-results">
- <div class="has-results">
-
- <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
- <ul class="search-results-list"></ul>
-
- </div>
- <div class="no-results">
-
- <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
-
- </div>
- </div>
-</div>
-
- </div>
- </div>
-
- </div>
-
-
-
-
- </div>
-
- <script>
- var gitbook = gitbook || [];
- gitbook.push(function() {
- gitbook.page.hasChanged({"page":{"title":"Feature selection","level":"3.5","depth":1,"next":{"title":"Statistical evaluation of a prediction model","level":"4.1","depth":1,"path":"eval/stat_eval.md","ref":"eval/stat_eval.md","articles":[{"title":"Area Under the ROC Curve","level":"4.1.1","depth":2,"path":"eval/auc.md","ref":"eval/auc.md","articles":[]}]},"previous":{"title":"Quantify non-number features","level":"3.4.2","depth":2,"path":"ft_engineering/quantify.md","ref":"ft_engineering/quantify.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{}
,"etoc":{"header":1,"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css"
,"mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"ft_engineering/feature_selection.md","mtime":"2016-12-13T12:22:11.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-04-27T13:49:22.144Z"},"basePath":"..","book":{"language":""}});
- });
- </script>
-</div>
-
-
- <script src="../gitbook/gitbook.js"></script>
- <script src="../gitbook/theme.js"></script>
-
-
- <script src="../gitbook/gitbook-plugin-edit-link/plugin.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-github/plugin.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-splitter/splitter.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-etoc/plugin.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-toggle-chapters/toggle.js"></script>
-
-
-
- <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/3.1.1/anchor.min.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-anchorjs/anchor-style.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-search/search.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
-
-
-
- <script src="../gitbook/gitbook-plugin-theme-api/theme-api.js"></script>
-
-
-
- </body>
-</html>
-
http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/15b9991a/userguide/ft_engineering/ft_trans.html
----------------------------------------------------------------------
diff --git a/userguide/ft_engineering/ft_trans.html b/userguide/ft_engineering/ft_trans.html
index ea1f85c..79f627e 100644
--- a/userguide/ft_engineering/ft_trans.html
+++ b/userguide/ft_engineering/ft_trans.html
@@ -97,7 +97,7 @@
<link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
- <link rel="next" href="vectorizer.html" />
+ <link rel="next" href="vectorization.html" />
<link rel="prev" href="tfidf.html" />
@@ -568,14 +568,14 @@
</li>
- <li class="chapter " data-level="3.3" data-path="tfidf.html">
+ <li class="chapter " data-level="3.3" data-path="selection.html">
- <a href="tfidf.html">
+ <a href="selection.html">
<b>3.3.</b>
- TF-IDF calculation
+ Feature Selection
</a>
@@ -583,30 +583,29 @@
</li>
- <li class="chapter active" data-level="3.4" data-path="ft_trans.html">
+ <li class="chapter " data-level="3.4" data-path="binning.html">
- <a href="ft_trans.html">
+ <a href="binning.html">
<b>3.4.</b>
- FEATURE TRANSFORMATION
+ Feature Binning
</a>
- <ul class="articles">
-
+ </li>
- <li class="chapter " data-level="3.4.1" data-path="vectorizer.html">
+ <li class="chapter " data-level="3.5" data-path="tfidf.html">
- <a href="vectorizer.html">
+ <a href="tfidf.html">
- <b>3.4.1.</b>
+ <b>3.5.</b>
- Vectorize Features
+ TF-IDF Calculation
</a>
@@ -614,34 +613,45 @@
</li>
- <li class="chapter " data-level="3.4.2" data-path="quantify.html">
+ <li class="chapter active" data-level="3.6" data-path="ft_trans.html">
- <a href="quantify.html">
+ <a href="ft_trans.html">
- <b>3.4.2.</b>
+ <b>3.6.</b>
- Quantify non-number features
+ FEATURE TRANSFORMATION
</a>
- </li>
+ <ul class="articles">
+
+ <li class="chapter " data-level="3.6.1" data-path="vectorization.html">
+
+ <a href="vectorization.html">
+
+
+ <b>3.6.1.</b>
+
+ Feature Vectorization
+
+ </a>
+
- </ul>
</li>
- <li class="chapter " data-level="3.5" data-path="feature_selection.html">
+ <li class="chapter " data-level="3.6.2" data-path="quantify.html">
- <a href="feature_selection.html">
+ <a href="quantify.html">
- <b>3.5.</b>
+ <b>3.6.2.</b>
- Feature selection
+ Quantify non-number features
</a>
@@ -650,6 +660,11 @@
</li>
+ </ul>
+
+ </li>
+
+
<li class="header">Part IV - Evaluation</li>
@@ -2047,7 +2062,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda
<script>
var gitbook = gitbook || [];
gitbook.push(function() {
- gitbook.page.hasChanged({"page":{"title":"FEATURE TRANSFORMATION","level":"3.4","depth":1,"next":{"title":"Vectorize Features","level":"3.4.1","depth":2,"path":"ft_engineering/vectorizer.md","ref":"ft_engineering/vectorizer.md","articles":[]},"previous":{"title":"TF-IDF calculation","level":"3.3","depth":1,"path":"ft_engineering/tfidf.md","ref":"ft_engineering/tfidf.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"header":1,"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitt
er":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,
*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"ft_engineering/ft_trans.md","mtime":"2016-12-02T08:02:42.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-04-27T13:49:22.144Z"},"basePath":"..","book":{"language":""}});
+ gitbook.page.hasChanged({"page":{"title":"FEATURE TRANSFORMATION","level":"3.6","depth":1,"next":{"title":"Feature Vectorization","level":"3.6.1","depth":2,"path":"ft_engineering/vectorization.md","ref":"ft_engineering/vectorization.md","articles":[]},"previous":{"title":"TF-IDF Calculation","level":"3.5","depth":1,"path":"ft_engineering/tfidf.md","ref":"ft_engineering/tfidf.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"header":1,"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"
},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"
h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"ft_engineering/ft_trans.md","mtime":"2017-05-08T04:55:36.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-05-08T08:42:42.226Z"},"basePath":"..","book":{"language":""}});
});
</script>
</div>