You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2016/10/22 19:23:20 UTC
[30/51] [partial] incubator-hivemall-site git commit: Added the user
guide
http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/30afb6e4/userguide/eval/lr_datagen.html
----------------------------------------------------------------------
diff --git a/userguide/eval/lr_datagen.html b/userguide/eval/lr_datagen.html
new file mode 100644
index 0000000..87ea306
--- /dev/null
+++ b/userguide/eval/lr_datagen.html
@@ -0,0 +1,1817 @@
+
+<!DOCTYPE HTML>
+<html lang="" >
+ <head>
+ <meta charset="UTF-8">
+ <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
+ <title>Logistic Regression data generation � Hivemall User Manual</title>
+ <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+ <meta name="description" content="">
+ <meta name="generator" content="GitBook 3.2.2">
+
+
+
+
+ <link rel="stylesheet" href="../gitbook/style.css">
+
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-splitter/splitter.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-etoc/plugin.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-callouts/plugin.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-toggle-chapters/toggle.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-codeblock-filename/block.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-multipart/multipart.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-katex/katex.min.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-emphasize/plugin.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
+
+
+
+ <link rel="stylesheet" href="../gitbook/gitbook-plugin-theme-api/theme-api.css">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <meta name="HandheldFriendly" content="true"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
+ <meta name="apple-mobile-web-app-capable" content="yes">
+ <meta name="apple-mobile-web-app-status-bar-style" content="black">
+ <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
+ <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">
+
+
+ <link rel="next" href="../binaryclass/a9a.html" />
+
+
+ <link rel="prev" href="datagen.html" />
+
+
+ </head>
+ <body>
+
+<div class="book">
+ <div class="book-summary">
+
+
+<div id="book-search-input" role="search">
+ <input type="text" placeholder="Type to search" />
+</div>
+
+
+ <nav role="navigation">
+
+
+
+<ul class="summary">
+
+
+
+
+ <li>
+ <a href="http://hivemall.incubator.apache.org/" target="_blank" class="custom-link"><i class="fa fa-home"></i> Home</a>
+ </li>
+
+
+
+
+ <li class="divider"></li>
+
+
+
+
+ <li class="header">TABLE OF CONTENTS</li>
+
+
+
+ <li class="chapter " data-level="1.1" data-path="../">
+
+ <a href="../">
+
+
+ <b>1.1.</b>
+
+ Introduction
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.2" data-path="../getting_started/">
+
+ <a href="../getting_started/">
+
+
+ <b>1.2.</b>
+
+ Getting Started
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="1.2.1" data-path="../getting_started/installation.html">
+
+ <a href="../getting_started/installation.html">
+
+
+ <b>1.2.1.</b>
+
+ Installation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.2.2" data-path="../getting_started/permanent-functions.html">
+
+ <a href="../getting_started/permanent-functions.html">
+
+
+ <b>1.2.2.</b>
+
+ Install as permanent functions
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.2.3" data-path="../getting_started/input-format.html">
+
+ <a href="../getting_started/input-format.html">
+
+
+ <b>1.2.3.</b>
+
+ Input Format
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="1.3" data-path="../tips/">
+
+ <a href="../tips/">
+
+
+ <b>1.3.</b>
+
+ Tips for Effective Hivemall
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="1.3.1" data-path="../tips/addbias.html">
+
+ <a href="../tips/addbias.html">
+
+
+ <b>1.3.1.</b>
+
+ Explicit addBias() for better prediction
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.2" data-path="../tips/rand_amplify.html">
+
+ <a href="../tips/rand_amplify.html">
+
+
+ <b>1.3.2.</b>
+
+ Use rand_amplify() to better prediction results
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.3" data-path="../tips/rt_prediction.html">
+
+ <a href="../tips/rt_prediction.html">
+
+
+ <b>1.3.3.</b>
+
+ Real-time Prediction on RDBMS
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.4" data-path="../tips/ensemble_learning.html">
+
+ <a href="../tips/ensemble_learning.html">
+
+
+ <b>1.3.4.</b>
+
+ Ensemble learning for stable prediction
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.5" data-path="../tips/mixserver.html">
+
+ <a href="../tips/mixserver.html">
+
+
+ <b>1.3.5.</b>
+
+ Mixing models for a better prediction convergence (MIX server)
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.3.6" data-path="../tips/emr.html">
+
+ <a href="../tips/emr.html">
+
+
+ <b>1.3.6.</b>
+
+ Run Hivemall on Amazon Elastic MapReduce
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="1.4" data-path="../tips/general_tips.html">
+
+ <a href="../tips/general_tips.html">
+
+
+ <b>1.4.</b>
+
+ General Hive/Hadoop tips
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="1.4.1" data-path="../tips/rowid.html">
+
+ <a href="../tips/rowid.html">
+
+
+ <b>1.4.1.</b>
+
+ Adding rowid for each row
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.4.2" data-path="../tips/hadoop_tuning.html">
+
+ <a href="../tips/hadoop_tuning.html">
+
+
+ <b>1.4.2.</b>
+
+ Hadoop tuning for Hivemall
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="1.5" data-path="../troubleshooting/">
+
+ <a href="../troubleshooting/">
+
+
+ <b>1.5.</b>
+
+ Troubleshooting
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="1.5.1" data-path="../troubleshooting/oom.html">
+
+ <a href="../troubleshooting/oom.html">
+
+
+ <b>1.5.1.</b>
+
+ OutOfMemoryError in training
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.5.2" data-path="../troubleshooting/mapjoin_task_error.html">
+
+ <a href="../troubleshooting/mapjoin_task_error.html">
+
+
+ <b>1.5.2.</b>
+
+ SemanticException Generate Map Join Task Error: Cannot serialize object
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.5.3" data-path="../troubleshooting/asterisk.html">
+
+ <a href="../troubleshooting/asterisk.html">
+
+
+ <b>1.5.3.</b>
+
+ Asterisk argument for UDTF does not work
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.5.4" data-path="../troubleshooting/num_mappers.html">
+
+ <a href="../troubleshooting/num_mappers.html">
+
+
+ <b>1.5.4.</b>
+
+ The number of mappers is less than input splits in Hadoop 2.x
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="1.5.5" data-path="../troubleshooting/mapjoin_classcastex.html">
+
+ <a href="../troubleshooting/mapjoin_classcastex.html">
+
+
+ <b>1.5.5.</b>
+
+ Map-side Join causes ClassCastException on Tez
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part II - Generic Features</li>
+
+
+
+ <li class="chapter " data-level="2.1" data-path="../misc/generic_funcs.html">
+
+ <a href="../misc/generic_funcs.html">
+
+
+ <b>2.1.</b>
+
+ List of generic Hivemall functions
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="2.2" data-path="../misc/topk.html">
+
+ <a href="../misc/topk.html">
+
+
+ <b>2.2.</b>
+
+ Efficient Top-K query processing
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="2.3" data-path="../misc/tokenizer.html">
+
+ <a href="../misc/tokenizer.html">
+
+
+ <b>2.3.</b>
+
+ English/Japanese Text Tokenizer
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part III - Feature Engineering</li>
+
+
+
+ <li class="chapter " data-level="3.1" data-path="../ft_engineering/scaling.html">
+
+ <a href="../ft_engineering/scaling.html">
+
+
+ <b>3.1.</b>
+
+ Feature Scaling
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="3.2" data-path="../ft_engineering/hashing.html">
+
+ <a href="../ft_engineering/hashing.html">
+
+
+ <b>3.2.</b>
+
+ Feature Hashing
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="3.3" data-path="../ft_engineering/tfidf.html">
+
+ <a href="../ft_engineering/tfidf.html">
+
+
+ <b>3.3.</b>
+
+ TF-IDF calculation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="3.4" data-path="../ft_engineering/ft_trans.html">
+
+ <a href="../ft_engineering/ft_trans.html">
+
+
+ <b>3.4.</b>
+
+ FEATURE TRANSFORMATION
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="3.4.1" data-path="../ft_engineering/vectorizer.html">
+
+ <a href="../ft_engineering/vectorizer.html">
+
+
+ <b>3.4.1.</b>
+
+ Vectorize Features
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="3.4.2" data-path="../ft_engineering/quantify.html">
+
+ <a href="../ft_engineering/quantify.html">
+
+
+ <b>3.4.2.</b>
+
+ Quantify non-number features
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part IV - Evaluation</li>
+
+
+
+ <li class="chapter " data-level="4.1" data-path="stat_eval.html">
+
+ <a href="stat_eval.html">
+
+
+ <b>4.1.</b>
+
+ Statistical evaluation of a prediction model
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="4.2" data-path="datagen.html">
+
+ <a href="datagen.html">
+
+
+ <b>4.2.</b>
+
+ Data Generation
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter active" data-level="4.2.1" data-path="lr_datagen.html">
+
+ <a href="lr_datagen.html">
+
+
+ <b>4.2.1.</b>
+
+ Logistic Regression data generation
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part V - Binary classification</li>
+
+
+
+ <li class="chapter " data-level="5.1" data-path="../binaryclass/a9a.html">
+
+ <a href="../binaryclass/a9a.html">
+
+
+ <b>5.1.</b>
+
+ a9a Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.1.1" data-path="../binaryclass/a9a_dataset.html">
+
+ <a href="../binaryclass/a9a_dataset.html">
+
+
+ <b>5.1.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.1.2" data-path="../binaryclass/a9a_lr.html">
+
+ <a href="../binaryclass/a9a_lr.html">
+
+
+ <b>5.1.2.</b>
+
+ Logistic Regression
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.1.3" data-path="../binaryclass/a9a_minibatch.html">
+
+ <a href="../binaryclass/a9a_minibatch.html">
+
+
+ <b>5.1.3.</b>
+
+ Mini-batch Gradient Descent
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="5.2" data-path="../binaryclass/news20.html">
+
+ <a href="../binaryclass/news20.html">
+
+
+ <b>5.2.</b>
+
+ News20 Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.2.1" data-path="../binaryclass/news20_dataset.html">
+
+ <a href="../binaryclass/news20_dataset.html">
+
+
+ <b>5.2.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.2.2" data-path="../binaryclass/news20_pa.html">
+
+ <a href="../binaryclass/news20_pa.html">
+
+
+ <b>5.2.2.</b>
+
+ Perceptron, Passive Aggressive
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.2.3" data-path="../binaryclass/news20_scw.html">
+
+ <a href="../binaryclass/news20_scw.html">
+
+
+ <b>5.2.3.</b>
+
+ CW, AROW, SCW
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.2.4" data-path="../binaryclass/news20_adagrad.html">
+
+ <a href="../binaryclass/news20_adagrad.html">
+
+
+ <b>5.2.4.</b>
+
+ AdaGradRDA, AdaGrad, AdaDelta
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="5.3" data-path="../binaryclass/kdd2010a.html">
+
+ <a href="../binaryclass/kdd2010a.html">
+
+
+ <b>5.3.</b>
+
+ KDD2010a Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.3.1" data-path="../binaryclass/kdd2010a_dataset.html">
+
+ <a href="../binaryclass/kdd2010a_dataset.html">
+
+
+ <b>5.3.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.3.2" data-path="../binaryclass/kdd2010a_scw.html">
+
+ <a href="../binaryclass/kdd2010a_scw.html">
+
+
+ <b>5.3.2.</b>
+
+ PA, CW, AROW, SCW
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="5.4" data-path="../binaryclass/kdd2010b.html">
+
+ <a href="../binaryclass/kdd2010b.html">
+
+
+ <b>5.4.</b>
+
+ KDD2010b Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.4.1" data-path="../binaryclass/kdd2010b_dataset.html">
+
+ <a href="../binaryclass/kdd2010b_dataset.html">
+
+
+ <b>5.4.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.4.2" data-path="../binaryclass/kdd2010b_arow.html">
+
+ <a href="../binaryclass/kdd2010b_arow.html">
+
+
+ <b>5.4.2.</b>
+
+ AROW
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="5.5" data-path="../binaryclass/webspam.html">
+
+ <a href="../binaryclass/webspam.html">
+
+
+ <b>5.5.</b>
+
+ Webspam Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="5.5.1" data-path="../binaryclass/webspam_dataset.html">
+
+ <a href="../binaryclass/webspam_dataset.html">
+
+
+ <b>5.5.1.</b>
+
+ Data pareparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="5.5.2" data-path="../binaryclass/webspam_scw.html">
+
+ <a href="../binaryclass/webspam_scw.html">
+
+
+ <b>5.5.2.</b>
+
+ PA1, AROW, SCW
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part VI - Multiclass classification</li>
+
+
+
+ <li class="chapter " data-level="6.1" data-path="../multiclass/news20.html">
+
+ <a href="../multiclass/news20.html">
+
+
+ <b>6.1.</b>
+
+ News20 Multiclass Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="6.1.1" data-path="../multiclass/news20_dataset.html">
+
+ <a href="../multiclass/news20_dataset.html">
+
+
+ <b>6.1.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.2" data-path="../multiclass/news20_one-vs-the-rest_dataset.html">
+
+ <a href="../multiclass/news20_one-vs-the-rest_dataset.html">
+
+
+ <b>6.1.2.</b>
+
+ Data preparation for one-vs-the-rest classifiers
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.3" data-path="../multiclass/news20_pa.html">
+
+ <a href="../multiclass/news20_pa.html">
+
+
+ <b>6.1.3.</b>
+
+ PA
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.4" data-path="../multiclass/news20_scw.html">
+
+ <a href="../multiclass/news20_scw.html">
+
+
+ <b>6.1.4.</b>
+
+ CW, AROW, SCW
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.5" data-path="../multiclass/news20_ensemble.html">
+
+ <a href="../multiclass/news20_ensemble.html">
+
+
+ <b>6.1.5.</b>
+
+ Ensemble learning
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.1.6" data-path="../multiclass/news20_one-vs-the-rest.html">
+
+ <a href="../multiclass/news20_one-vs-the-rest.html">
+
+
+ <b>6.1.6.</b>
+
+ one-vs-the-rest classifier
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="6.2" data-path="../multiclass/iris.html">
+
+ <a href="../multiclass/iris.html">
+
+
+ <b>6.2.</b>
+
+ Iris Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="6.2.1" data-path="../multiclass/iris_dataset.html">
+
+ <a href="../multiclass/iris_dataset.html">
+
+
+ <b>6.2.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.2.2" data-path="../multiclass/iris_scw.html">
+
+ <a href="../multiclass/iris_scw.html">
+
+
+ <b>6.2.2.</b>
+
+ SCW
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="6.2.3" data-path="../multiclass/iris_randomforest.html">
+
+ <a href="../multiclass/iris_randomforest.html">
+
+
+ <b>6.2.3.</b>
+
+ RandomForest
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part VII - Regression</li>
+
+
+
+ <li class="chapter " data-level="7.1" data-path="../regression/e2006.html">
+
+ <a href="../regression/e2006.html">
+
+
+ <b>7.1.</b>
+
+ E2006-tfidf regression Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="7.1.1" data-path="../regression/e2006_dataset.html">
+
+ <a href="../regression/e2006_dataset.html">
+
+
+ <b>7.1.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="7.1.2" data-path="../regression/e2006_arow.html">
+
+ <a href="../regression/e2006_arow.html">
+
+
+ <b>7.1.2.</b>
+
+ Passive Aggressive, AROW
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="7.2" data-path="../regression/kddcup12tr2.html">
+
+ <a href="../regression/kddcup12tr2.html">
+
+
+ <b>7.2.</b>
+
+ KDDCup 2012 track 2 CTR prediction Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="7.2.1" data-path="../regression/kddcup12tr2_dataset.html">
+
+ <a href="../regression/kddcup12tr2_dataset.html">
+
+
+ <b>7.2.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="7.2.2" data-path="../regression/kddcup12tr2_lr.html">
+
+ <a href="../regression/kddcup12tr2_lr.html">
+
+
+ <b>7.2.2.</b>
+
+ Logistic Regression, Passive Aggressive
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="7.2.3" data-path="../regression/kddcup12tr2_lr_amplify.html">
+
+ <a href="../regression/kddcup12tr2_lr_amplify.html">
+
+
+ <b>7.2.3.</b>
+
+ Logistic Regression with Amplifier
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="7.2.4" data-path="../regression/kddcup12tr2_adagrad.html">
+
+ <a href="../regression/kddcup12tr2_adagrad.html">
+
+
+ <b>7.2.4.</b>
+
+ AdaGrad, AdaDelta
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part VIII - Recommendation</li>
+
+
+
+ <li class="chapter " data-level="8.1" data-path="../recommend/cf.html">
+
+ <a href="../recommend/cf.html">
+
+
+ <b>8.1.</b>
+
+ Collaborative Filtering
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="8.1.1" data-path="../recommend/item_based_cf.html">
+
+ <a href="../recommend/item_based_cf.html">
+
+
+ <b>8.1.1.</b>
+
+ Item-based Collaborative Filtering
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="8.2" data-path="../recommend/news20.html">
+
+ <a href="../recommend/news20.html">
+
+
+ <b>8.2.</b>
+
+ News20 related article recommendation Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="8.2.1" data-path="../multiclass/news20_dataset.html">
+
+ <a href="../multiclass/news20_dataset.html">
+
+
+ <b>8.2.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.2.2" data-path="../recommend/news20_jaccard.html">
+
+ <a href="../recommend/news20_jaccard.html">
+
+
+ <b>8.2.2.</b>
+
+ LSH/Minhash and Jaccard Similarity
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.2.3" data-path="../recommend/news20_knn.html">
+
+ <a href="../recommend/news20_knn.html">
+
+
+ <b>8.2.3.</b>
+
+ LSH/Minhash and Brute-Force Search
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.2.4" data-path="../recommend/news20_bbit_minhash.html">
+
+ <a href="../recommend/news20_bbit_minhash.html">
+
+
+ <b>8.2.4.</b>
+
+ kNN search using b-Bits Minhash
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+ <li class="chapter " data-level="8.3" data-path="../recommend/movielens.html">
+
+ <a href="../recommend/movielens.html">
+
+
+ <b>8.3.</b>
+
+ MovieLens movie recommendation Tutorial
+
+ </a>
+
+
+
+ <ul class="articles">
+
+
+ <li class="chapter " data-level="8.3.1" data-path="../recommend/movielens_dataset.html">
+
+ <a href="../recommend/movielens_dataset.html">
+
+
+ <b>8.3.1.</b>
+
+ Data preparation
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.3.2" data-path="../recommend/movielens_mf.html">
+
+ <a href="../recommend/movielens_mf.html">
+
+
+ <b>8.3.2.</b>
+
+ Matrix Factorization
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.3.3" data-path="../recommend/movielens_fm.html">
+
+ <a href="../recommend/movielens_fm.html">
+
+
+ <b>8.3.3.</b>
+
+ Factorization Machine
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="8.3.4" data-path="../recommend/movielens_cv.html">
+
+ <a href="../recommend/movielens_cv.html">
+
+
+ <b>8.3.4.</b>
+
+ 10-fold Cross Validation (Matrix Factorization)
+
+ </a>
+
+
+
+ </li>
+
+
+ </ul>
+
+ </li>
+
+
+
+
+ <li class="header">Part IX - Anomaly Detection</li>
+
+
+
+ <li class="chapter " data-level="9.1" data-path="../anomaly/lof.html">
+
+ <a href="../anomaly/lof.html">
+
+
+ <b>9.1.</b>
+
+ Outlier Detection using Local Outlier Factor (LOF)
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="header">Part X - External References</li>
+
+
+
+ <li class="chapter " data-level="10.1" >
+
+ <a target="_blank" href="https://github.com/maropu/hivemall-spark">
+
+
+ <b>10.1.</b>
+
+ Hivemall on Apache Spark
+
+ </a>
+
+
+
+ </li>
+
+ <li class="chapter " data-level="10.2" >
+
+ <a target="_blank" href="https://github.com/daijyc/hivemall/wiki/PigHome">
+
+
+ <b>10.2.</b>
+
+ Hivemall on Apache Pig
+
+ </a>
+
+
+
+ </li>
+
+
+
+
+ <li class="divider"></li>
+
+ <li>
+ <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
+ Published with GitBook
+ </a>
+ </li>
+</ul>
+
+
+ </nav>
+
+
+ </div>
+
+ <div class="book-body">
+
+ <div class="body-inner">
+
+
+
+<div class="book-header" role="navigation">
+
+
+ <!-- Title -->
+ <h1>
+ <i class="fa fa-circle-o-notch fa-spin"></i>
+ <a href=".." >Logistic Regression data generation</a>
+ </h1>
+</div>
+
+
+
+
+ <div class="page-wrapper" tabindex="-1" role="main">
+ <div class="page-inner">
+
+<div id="book-search-results">
+ <div class="search-noresults">
+
+ <section class="normal markdown-section">
+
+ <p><em>Note this feature is supported on hivemall v0.2-alpha3 or later.</em></p>
+<h1 id="create-a-dual-table">create a dual table</h1>
+<p>Create a <a href="http://en.wikipedia.org/wiki/DUAL_table" target="_blank">dual table</a> as follows:</p>
+<pre><code class="lang-sql"><span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">TABLE</span> dual (
+ dummy <span class="hljs-built_in">int</span>
+);
+<span class="hljs-keyword">INSERT</span> <span class="hljs-keyword">INTO</span> <span class="hljs-keyword">TABLE</span> dual <span class="hljs-keyword">SELECT</span> <span class="hljs-keyword">count</span>(*)+<span class="hljs-number">1</span> <span class="hljs-keyword">FROM</span> dual;
+</code></pre>
+<h1 id="sparse-dataset-generation-by-a-single-task">Sparse dataset generation by a single task</h1>
+<pre><code class="lang-sql"><span class="hljs-keyword">create</span> <span class="hljs-keyword">table</span> regression_data1
+<span class="hljs-keyword">as</span>
+<span class="hljs-keyword">select</span> lr_datagen(<span class="hljs-string">"-n_examples 10k -n_features 10 -seed 100"</span>) <span class="hljs-keyword">as</span> (label,features)
+<span class="hljs-keyword">from</span> dual;
+</code></pre>
+<p>Find the details of the option in <a href="https://github.com/myui/hivemall/blob/master/core/src/main/java/hivemall/dataset/LogisticRegressionDataGeneratorUDTF.java#L69" target="_blank">LogisticRegressionDataGeneratorUDTF.java</a>.</p>
+<p>You can generate a sparse dataset as well as a dense dataset. By the default, a sparse dataset is generated.</p>
+<pre><code class="lang-sql">hive> desc regression_data1;
+OK
+label float None
+features array<string> None
+
+hive> select * from regression_data1 limit 2;
+OK
+0.7220096 ["140:2.8347101","165:3.0056276","179:4.030076","112:3.3919246","99:3.98914","16:3.5653272","128:3.046535","124:2.7708225","78:2.4960368","6:1.7866131"]
+0.7346627 ["139:1.9607254","110:2.958568","186:3.2524762","31:3.9243593","167:0.72854257","26:1.8355447","117:2.7663715","3:2.1551287","179:3.1099443","19:3.6411424"]
+Time taken: 0.046 seconds, Fetched: 2 row(s)
+</code></pre>
+<h1 id="classification-dataset-generation">Classification dataset generation</h1>
+<p>You can use "-cl" option to generation 0/1 label.</p>
+<pre><code class="lang-sql"><span class="hljs-keyword">select</span> lr_datagen(<span class="hljs-string">"-cl"</span>) <span class="hljs-keyword">as</span> (label,features)
+<span class="hljs-keyword">from</span> dual
+<span class="hljs-keyword">limit</span> <span class="hljs-number">5</span>;
+OK
+1.0 ["84:3.4227803","80:3.8875976","58:3.2909582","123:3.1056073","194:3.3360343","199:2.20207","75:3.5469763","74:3.3869767","126:0.9969454","93:2.5352612"]
+0.0 ["84:-0.5568947","10:0.621897","6:-0.13126314","190:0.18610542","131:1.7232913","24:-2.7551131","113:-0.9842969","177:0.062993184","176:-0.19020283","21:-0.54811275"]
+1.0 ["73:3.4391513","198:4.42387","164:4.248151","66:3.5224934","84:1.9026604","76:0.79803777","18:2.2168183","163:2.248695","119:1.5906067","72:2.0267224"]
+1.0 ["34:2.9269936","35:0.37033868","39:3.771989","47:2.2087111","28:2.9445739","55:4.134555","14:2.4297745","164:3.0913055","52:2.0519433","128:2.9108515"]
+1.0 ["98:4.2451696","4:3.486905","133:2.4589922","26:2.7301126","103:2.6827147","2:3.6198254","34:3.7042716","47:2.5515237","68:2.4294896","197:4.4958663"]
+</code></pre>
+<h1 id="dense-dataset-generation">Dense dataset generation</h1>
+<pre><code class="lang-sql">create table regression_data_dense
+as
+select lr_datagen("-dense -n_examples 9999 -n_features 100 -n_dims 100") as (label,features)
+from dual;
+
+hive> desc regression_data_dense;
+OK
+label float None
+features array<float> None
+
+hive> select * from regression_data_dense limit 1;
+OK
+0.7274741 [4.061373,3.9373128,3.5195694,3.3604698,3.7698417,4.2518,3.8796813,1.6020582,4.937072,1.5513933,3.0289552,2.6674519,3.432688,2.980945,1.8897587,2.9770515,3.3435504,1.7867403,3.4057906,1.2151588,5.0587463,2.1410913,2.8097973,2.4518871,3.175268,3.3347685,3.728993,3.1443396,3.5506077,3.6357877,4.248151,3.5224934,3.2423255,2.5188355,1.8626233,2.8432152,2.2762651,4.57472,2.2168183,2.248695,3.3636255,2.8359523,2.0327945,1.5917025,2.9269936,0.37033868,2.6151125,4.545956,2.0863252,3.7857852,2.9445739,4.134555,3.0660007,3.4279037,2.0519433,2.9108515,3.5171766,3.4708095,3.161707,2.39229,2.4589922,2.7301126,3.5303073,2.7398396,3.7042716,2.5515237,3.0943663,0.41565156,4.672767,3.1461313,3.0443575,3.4023938,2.2205734,1.8950733,2.1664586,4.8654623,2.787029,4.0460386,2.4455893,3.464298,1.062505,3.0513604,4.382525,2.771433,3.2828436,3.803544,2.178681,4.2466116,3.5440445,3.1546876,3.4248536,0.9067459,3.0134914,1.9528451,1.7175893,2.7029774,2.5759792,3.643847,3.0799,3.735559]
+Time taken: 0.044 seconds, Fetched: 1 row(s)
+</code></pre>
+<h1 id="parallel-and-scalable-data-generation-using-multiple-reducers-recommended">Parallel and scalable data generation using multiple reducers (RECOMMENDED)</h1>
+<p>Dataset generation using (at max) 10 reducers.</p>
+<pre><code class="lang-sql">set hivevar:n_parallel_datagen=10;
+
+create or replace view seq10
+as
+select * from (
+ select generate_series(1,${n_parallel_datagen})
+ from dual
+) t
+DISTRIBUTE BY value;
+
+set mapred.reduce.tasks=${n_parallel_datagen};
+create table lrdata1k
+as
+select lr_datagen("-n_examples 100")
+from seq10;
+set mapred.reduce.tasks=-1; -- reset to the default setting
+
+hive> select count(1) from lrdata1k;
+OK
+1000
+</code></pre>
+
+
+ </section>
+
+ </div>
+ <div class="search-results">
+ <div class="has-results">
+
+ <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
+ <ul class="search-results-list"></ul>
+
+ </div>
+ <div class="no-results">
+
+ <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
+
+ </div>
+ </div>
+</div>
+
+ </div>
+ </div>
+
+ </div>
+
+
+
+
+ </div>
+
+ <script>
+ var gitbook = gitbook || [];
+ gitbook.push(function() {
+ gitbook.page.hasChanged({"page":{"title":"Logistic Regression data generation","level":"4.2.1","depth":2,"next":{"title":"a9a Tutorial","level":"5.1","depth":1,"path":"binaryclass/a9a.md","ref":"binaryclass/a9a.md","articles":[{"title":"Data preparation","level":"5.1.1","depth":2,"path":"binaryclass/a9a_dataset.md","ref":"binaryclass/a9a_dataset.md","articles":[]},{"title":"Logistic Regression","level":"5.1.2","depth":2,"path":"binaryclass/a9a_lr.md","ref":"binaryclass/a9a_lr.md","articles":[]},{"title":"Mini-batch Gradient Descent","level":"5.1.3","depth":2,"path":"binaryclass/a9a_minibatch.md","ref":"binaryclass/a9a_minibatch.md","articles":[]}]},"previous":{"title":"Data Generation","level":"4.2","depth":1,"path":"eval/datagen.md","ref":"eval/datagen.md","articles":[{"title":"Logistic Regression data generation","level":"4.2.1","depth":2,"path":"eval/lr_datagen.md","ref":"eval/lr_datagen.md","articles":[]}]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","g
ithub","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twi
tter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x
","description":"User Manual for Apache Hivemall"},"file":{"path":"eval/lr_datagen.md","mtime":"2016-10-22T17:12:43.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2016-10-22T19:10:36.585Z"},"basePath":"..","book":{"language":""}});
+ });
+ </script>
+</div>
+
+
+ <script src="../gitbook/gitbook.js"></script>
+ <script src="../gitbook/theme.js"></script>
+
+
+ <script src="../gitbook/gitbook-plugin-edit-link/plugin.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-github/plugin.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-splitter/splitter.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-etoc/plugin.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-toggle-chapters/toggle.js"></script>
+
+
+
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/3.1.1/anchor.min.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-anchorjs/anchor-style.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-search/search.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
+
+
+
+ <script src="../gitbook/gitbook-plugin-theme-api/theme-api.js"></script>
+
+
+
+ </body>
+</html>
+