You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2019/11/27 07:42:41 UTC
[incubator-hivemall] branch master updated: [HIVEMALL-158][DOC]
Refine deprecated userguide contents
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
The following commit(s) were added to refs/heads/master by this push:
new ab92ecc [HIVEMALL-158][DOC] Refine deprecated userguide contents
ab92ecc is described below
commit ab92ecca43418395293da030803f868cdb27ee20
Author: Makoto Yui <my...@apache.org>
AuthorDate: Wed Nov 27 16:42:34 2019 +0900
[HIVEMALL-158][DOC] Refine deprecated userguide contents
## What changes were proposed in this pull request?
Refine deprecated userguide contents
## What type of PR is it?
Documentation
## What is the Jira issue?
https://issues.apache.org/jira/browse/HIVEMALL-158
Author: Makoto Yui <my...@apache.org>
Closes #221 from myui/HIVEMALL-158.
---
docs/gitbook/regression/e2006_arow.md | 25 +++++--
docs/gitbook/regression/kddcup12tr2_adagrad.md | 30 ++++----
docs/gitbook/regression/kddcup12tr2_dataset.md | 99 +++++++++++---------------
docs/gitbook/regression/kddcup12tr2_lr.md | 41 +++++++----
4 files changed, 105 insertions(+), 90 deletions(-)
diff --git a/docs/gitbook/regression/e2006_arow.md b/docs/gitbook/regression/e2006_arow.md
index 1342f8f..9a341f6 100644
--- a/docs/gitbook/regression/e2006_arow.md
+++ b/docs/gitbook/regression/e2006_arow.md
@@ -22,6 +22,7 @@
# PA1a
## Training
+
```sql
set mapred.reduce.tasks=64;
@@ -46,6 +47,7 @@ set mapred.reduce.tasks=-1;
> Do not use `voted_avg()` for regression. `voted_avg()` is for classification.
## prediction
+
```sql
create or replace view e2006tfidf_pa1a_predict
as
@@ -60,6 +62,7 @@ group by
```
## evaluation
+
```sql
WITH submit as (
select
@@ -86,6 +89,7 @@ from
# PA2a
## Training
+
```sql
set mapred.reduce.tasks=64;
drop table e2006tfidf_pa2a_model;
@@ -104,6 +108,7 @@ set mapred.reduce.tasks=-1;
```
## prediction
+
```sql
create or replace view e2006tfidf_pa2a_predict
as
@@ -118,6 +123,7 @@ group by
```
## evaluation
+
```sql
WITH submit as (
select
@@ -144,6 +150,7 @@ from
# AROW
## Training
+
```sql
set mapred.reduce.tasks=64;
drop table e2006tfidf_arow_model ;
@@ -154,8 +161,7 @@ select
argmin_kld(weight, covar) as weight -- [hivemall v0.2 or later]
from
(select
- -- train_arow_regr(add_bias(features),target) as (feature,weight) -- [hivemall v0.1]
- train_arow_regr(add_bias(features),target) as (feature,weight,covar) -- [hivemall v0.2 or later]
+ train_arow_regr(add_bias(features),target) as (feature,weight,covar)
from
e2006tfidf_train_x3
) t
@@ -164,6 +170,7 @@ set mapred.reduce.tasks=-1;
```
## prediction
+
```sql
create or replace view e2006tfidf_arow_predict
as
@@ -178,6 +185,7 @@ group by
```
## evaluation
+
```sql
WITH submit as (
select
@@ -202,21 +210,22 @@ from
| 0.37862513029019407 | 0.14335698928726642 | 0.2368787001269389 | 0.5041085155590119 |
# AROWe
+
AROWe is a modified version of AROW that uses Hinge loss (epsilion = 0.1)
## Training
+
```sql
-set mapred.reduce.tasks=64;
+-- set mapred.reduce.tasks=64;
+
drop table e2006tfidf_arowe_model ;
create table e2006tfidf_arowe_model as
select
feature,
- -- avg(weight) as weight -- [hivemall v0.1]
- argmin_kld(weight, covar) as weight -- [hivemall v0.2 or later]
+ argmin_kld(weight, covar) as weight
from
(select
- -- train_arowe_regr(add_bias(features),target) as (feature,weight) -- [hivemall v0.1]
- train_arowe_regr(add_bias(features),target) as (feature,weight,covar) -- [hivemall v0.2 or later]
+ train_arowe_regr(add_bias(features),target) as (feature,weight,covar)
from
e2006tfidf_train_x3
) t
@@ -225,6 +234,7 @@ set mapred.reduce.tasks=-1;
```
## prediction
+
```sql
create or replace view e2006tfidf_arowe_predict
as
@@ -239,6 +249,7 @@ group by
```
## evaluation
+
```sql
WITH submit as (
select
diff --git a/docs/gitbook/regression/kddcup12tr2_adagrad.md b/docs/gitbook/regression/kddcup12tr2_adagrad.md
index 1b82bd9..a245428 100644
--- a/docs/gitbook/regression/kddcup12tr2_adagrad.md
+++ b/docs/gitbook/regression/kddcup12tr2_adagrad.md
@@ -16,14 +16,10 @@
specific language governing permissions and limitations
under the License.
-->
-
-_Note adagrad/adadelta is supported from hivemall v0.3b2 or later (or in the master branch)._
-# Preparation
-```sql
-add jar ./tmp/hivemall-with-dependencies.jar;
-source ./tmp/define-all.hive;
+# Preparation
+```sql
use kdd12track2;
-- SET mapreduce.framework.name=yarn;
@@ -34,6 +30,7 @@ SET mapred.reduce.tasks=32; -- [optional] set the explicit number of reducers to
```
# AdaGrad
+
```sql
drop table adagrad_model;
create table adagrad_model
@@ -60,7 +57,7 @@ select
t.rowid,
sigmoid(sum(m.weight)) as prob
from
- testing_exploded t LEFT OUTER JOIN
+ testing_exploded t LEFT OUTER JOIN
adagrad_model m ON (t.feature = m.feature)
group by
t.rowid
@@ -68,6 +65,9 @@ order by
rowid ASC;
```
+> #### Note
+> `sigmoid(sum(m.weight))` not `sigmoid(sum(m.weight * t.value)))` because t.value is always 1.0 for categorical variable.
+
```sh
hadoop fs -getmerge /user/hive/warehouse/kdd12track2.db/adagrad_predict adagrad_predict.tbl
@@ -75,11 +75,14 @@ gawk -F "\t" '{print $2;}' adagrad_predict.tbl > adagrad_predict.submit
pypy scoreKDD.py KDD_Track2_solution.csv adagrad_predict.submit
```
->AUC(SGD) : 0.739351
->AUC(ADAGRAD) : 0.743279
+|Algorithm|AUC|
+|:-:|:-:|
+| SGD | 0.739351 |
+| ADAGRAD | 0.743279 |
# AdaDelta
+
```sql
drop table adadelta_model;
create table adadelta_model
@@ -121,8 +124,9 @@ gawk -F "\t" '{print $2;}' adadelta_predict.tbl > adadelta_predict.submit
pypy scoreKDD.py KDD_Track2_solution.csv adadelta_predict.submit
```
->AUC(SGD) : 0.739351
-
->AUC(ADAGRAD) : 0.743279
-> AUC(AdaDelta) : 0.746878
+|Algorithm|AUC|
+|:-:|:-:|
+| SGD | 0.739351 |
+| ADAGRAD | 0.743279 |
+| AdaDelta | 0.746878 |
diff --git a/docs/gitbook/regression/kddcup12tr2_dataset.md b/docs/gitbook/regression/kddcup12tr2_dataset.md
index 8d512cd..bcdf18c 100644
--- a/docs/gitbook/regression/kddcup12tr2_dataset.md
+++ b/docs/gitbook/regression/kddcup12tr2_dataset.md
@@ -45,10 +45,6 @@ _Tokens are actually not used in this example. Try using them on your own._
create database kdd12track2;
use kdd12track2;
-delete jar /tmp/hivemall.jar;
-add jar /tmp/hivemall.jar;
-source /tmp/define-all.hive;
-
Create external table training (
RowID BIGINT,
Clicks INT,
@@ -156,14 +152,14 @@ select
mhash(concat("12:", COALESCE(age,"-1"))) as age,
-1 as bias
from (
-select
- t.*,
- u.gender,
- u.age
-from
- training t
- LEFT OUTER JOIN user u
- on t.userid = u.userid
+ select
+ t.*,
+ u.gender,
+ u.age
+ from
+ training t
+ LEFT OUTER JOIN user u
+ on t.userid = u.userid
) t;
create or replace view testing2 as
@@ -172,31 +168,31 @@ select
array(displayurl, adid, advertiserid, depth, position, queryid, keywordid, titleid, descriptionid, userid, gender, age, bias)
as features
from (
-select
- rowid,
- mhash(concat("1:", displayurl)) as displayurl,
- mhash(concat("2:", adid)) as adid,
- mhash(concat("3:", advertiserid)) as advertiserid,
- mhash(concat("4:", depth)) as depth,
- mhash(concat("5:", position)) as position,
- mhash(concat("6:", queryid)) as queryid,
- mhash(concat("7:", keywordid)) as keywordid,
- mhash(concat("8:", titleid)) as titleid,
- mhash(concat("9:", descriptionid)) as descriptionid,
- mhash(concat("10:", userid)) as userid,
- mhash(concat("11:", COALESCE(gender,"0"))) as gender,
- mhash(concat("12:", COALESCE(age,"-1"))) as age,
- -1 as bias
-from (
-select
- t.*,
- u.gender,
- u.age
-from
- testing t
- LEFT OUTER JOIN user u
- on t.userid = u.userid
-) t1
+ select
+ rowid,
+ mhash(concat("1:", displayurl)) as displayurl,
+ mhash(concat("2:", adid)) as adid,
+ mhash(concat("3:", advertiserid)) as advertiserid,
+ mhash(concat("4:", depth)) as depth,
+ mhash(concat("5:", position)) as position,
+ mhash(concat("6:", queryid)) as queryid,
+ mhash(concat("7:", keywordid)) as keywordid,
+ mhash(concat("8:", titleid)) as titleid,
+ mhash(concat("9:", descriptionid)) as descriptionid,
+ mhash(concat("10:", userid)) as userid,
+ mhash(concat("11:", COALESCE(gender,"0"))) as gender,
+ mhash(concat("12:", COALESCE(age,"-1"))) as age,
+ -1 as bias
+ from (
+ select
+ t.*,
+ u.gender,
+ u.age
+ from
+ testing t
+ LEFT OUTER JOIN user u
+ on t.userid = u.userid
+ ) t1
) t2;
```
@@ -204,33 +200,24 @@ from
```sql
create table training_orcfile (
rowid bigint,
- label float,
- features array<int>
+ features array<int>,
+ label int
) STORED AS orc tblproperties ("orc.compress"="SNAPPY");
```
-_Caution: Joining between training table and user table takes a long time. Consider not to use gender and age and avoid joins if your Hadoop cluster is small._
-[kddconv.awk](https://github.com/apache/incubator-hivemall/blob/master/resources/examples/kddtrack2/kddconv.awk)
+> _Caution: Joining between training table and user table takes a long time. Consider not to use gender and age and avoid joins if your Hadoop cluster is small._
```sql
-add file /tmp/kddconv.awk;
-
-- SET mapred.reduce.tasks=64;
-- SET hive.auto.convert.join=false;
INSERT OVERWRITE TABLE training_orcfile
-select transform(*)
- ROW FORMAT DELIMITED
- FIELDS TERMINATED BY "\t"
- LINES TERMINATED BY "\n"
-using 'gawk -f kddconv.awk'
- as (rowid BIGINT, label FLOAT, features ARRAY<INT>)
- ROW FORMAT DELIMITED
- FIELDS TERMINATED BY "\t"
- COLLECTION ITEMS TERMINATED BY ","
- LINES TERMINATED BY "\n"
-from training2
-CLUSTER BY rand();
+select
+ binarize_label(clicks, noclick, rowid, features)
+ as (rowid, features, label)
+from
+ training2
+CLUSTER BY rand(); -- shuffle
-- SET mapred.reduce.tasks=-1;
-- SET hive.auto.convert.join=true;
@@ -243,4 +230,4 @@ from
testing2
LATERAL VIEW explode(features) t AS feature;
```
-_Caution: We recommend you to set "mapred.reduce.tasks" in the above example to partition the training_orcfile table into pieces._
+
diff --git a/docs/gitbook/regression/kddcup12tr2_lr.md b/docs/gitbook/regression/kddcup12tr2_lr.md
index 42b8141..35f087c 100644
--- a/docs/gitbook/regression/kddcup12tr2_lr.md
+++ b/docs/gitbook/regression/kddcup12tr2_lr.md
@@ -22,9 +22,9 @@ https://www.kaggle.com/c/kddcup2012-track2
_Caution: This example just shows a baseline result. Use token tables and amplifier to get better AUC score._
----
-Logistic Regression
-===============
+<!-- toc -->
+
+# Logistic Regression
## Training
```sql
@@ -59,10 +59,13 @@ group by feature;
-- set mapred.max.split.size=-1; -- reset to the default value
```
-_Note: Setting the "-total_steps" option is optional._
+
+> #### Note
+> Setting the "-total_steps" option is optional.
## Prediction
-```
+
+```sql
drop table lr_predict;
create table lr_predict
ROW FORMAT DELIMITED
@@ -74,13 +77,18 @@ select
t.rowid,
sigmoid(sum(m.weight)) as prob
from
- testing_exploded t LEFT OUTER JOIN
- lr_model m ON (t.feature = m.feature)
+ testing_exploded t
+ LEFT OUTER JOIN lr_model m
+ ON (t.feature = m.feature)
group by
t.rowid
order by
rowid ASC;
```
+
+> #### Note
+> `sigmoid(sum(m.weight))` not `sigmoid(sum(m.weight * t.value)))` because t.value is always 1.0 for categorical variable.
+
## Evaluation
You can download scoreKDD.py from [KDD Cup 2012, Track 2 site](https://www.kaggle.com/c/kddcup2012-track2/data). After logging-in to Kaggle, download
@@ -93,7 +101,6 @@ gawk -F "\t" '{print $2;}' lr_predict.tbl > lr_predict.submit
pypy scoreKDD.py KDD_Track2_solution.csv lr_predict.submit
```
-_Note: You can use python instead of pypy._
| Measure | Score |
|:-----------|------------:|
@@ -101,11 +108,12 @@ _Note: You can use python instead of pypy._
| NWMAE | 0.045493 |
| WRMSE | 0.142395 |
---
-Passive Aggressive
-===============
+
+# Passive Aggressive
## Training
-```
+
+```sql
drop table pa_model;
create table pa_model
as
@@ -120,10 +128,13 @@ from
) t
group by feature;
```
-_PA1a is recommended when using PA for regression._
+
+> #### Note
+> PA1a is recommended when using PA for regression.
## Prediction
-```
+
+```sql
drop table pa_predict;
create table pa_predict
ROW FORMAT DELIMITED
@@ -142,7 +153,9 @@ group by
order by
rowid ASC;
```
-_The "prob" of PA can be used only for ranking and can have a negative value. A higher weight means much likely to be clicked. Note that AUC is sort a measure for evaluating ranking accuracy._
+
+> #### Caution
+> The "prob" of PA can be used only for ranking and can have a negative value. A higher weight means much likely to be clicked. Note that AUC is sort a measure for evaluating ranking accuracy.
## Evaluation