You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by on...@apache.org on 2014/07/16 18:09:22 UTC
[2/5] AMBARI-6507. Create mappers for models. (onechiporenko)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b76efd5b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/hive-query-2.json
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/hive-query-2.json b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/hive-query-2.json
new file mode 100644
index 0000000..90e6ccd
--- /dev/null
+++ b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/hive-query-2.json
@@ -0,0 +1,8 @@
+{
+ "starttime": 1394569191001,
+ "otherinfo": {
+ "query": "{\"queryText\":\"select c_last_name, c_first_name, ca_city, bought_city,\\n ss_ticket_number, amt, profit\\n from (select ss_ticket_number,\\n ss_customer_sk,\\n ca_city as bought_city,\\n sum(ss_coupon_amt) as amt,\\n sum(ss_net_profit) as profit\\n from store_sales, date_dim, store, household_demographics, customer_address\\n where store_sales.ss_sold_date_sk = date_dim.d_date_sk\\n and store_sales.ss_store_sk = store.s_store_sk\\n and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk\\n and store_sales.ss_addr_sk = customer_address.ca_address_sk\\n and (househo
ld_demographics.hd_dep_count = 4\\n or household_demographics.hd_vehicle_count= 3)\\n and date_dim.d_dow in (6,0)\\n and date_dim.d_year in (1999, 1999+1, 1999+2)\\n and store.s_city in ('Fairview', 'Fairview', 'Fairview', 'Midway', 'Fairview')\\n group by ss_ticket_number, ss_customer_sk, ss_addr_sk, ca_city) dn,\\n customer, customer_address current_addr\\n where dn.ss_customer_sk = customer.c_customer_sk\\n and customer.c_current_addr_sk = current_addr.ca_address_sk\\n and current_addr.ca_city <> dn.bought_city\\n order by c_last_name, c_first_name, ca_city, bought_city, ss_ticket_number\\n limit 100\",\"queryPlan\":{\"STAGE PLANS\":{\"Stage-1\":{\"Tez\":{\"
DagName:\":\"hrt_qa_20140311131919_1d932567-71c2-4341-9b50-6df1f58a9114:1\",\"Vertices:\":{\"Reducer 3\":{\"Reduce Operator Tree:\":{\"Group By Operator\":{\"mode:\":\"mergepartial\",\"aggregations:\":[\"sum(VALUE._col0)\",\"sum(VALUE._col1)\"],\"keys:\":\"KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int), KEY._col3 (type: string)\",\"outputColumnNames:\":[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\"],\"children\":{\"Select Operator\":{\"expressions:\":\"_col0 (type: int), _col1 (type: int), _col3 (type: string), _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2))\",\"outputColumnNames:\":[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\"],\"children\":{\"Reduce Output Operator\":{\"Map-reduce partition columns:\":\"_col1 (type: int)\",\"sort order:\":\"+\",\"value expressions:\":\"_col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: decimal(17,2)), _col4 (type: decimal(17,2))\",\"Statistics:\":\"Num rows: 16473 Data size: 21
775806 Basic stats: COMPLETE Column stats: NONE\",\"key expressions:\":\"_col1 (type: int)\"}},\"Statistics:\":\"Num rows: 16473 Data size: 21775806 Basic stats: COMPLETE Column stats: NONE\"}},\"Statistics:\":\"Num rows: 16473 Data size: 21775806 Basic stats: COMPLETE Column stats: NONE\"}}},\"Map 1\":{\"Execution mode:\":\"vectorized\",\"Map Operator Tree:\":[{\"TableScan\":{\"filterExpr:\":\"((hd_dep_count = 4) or (hd_vehicle_count = 3)) (type: boolean)\",\"alias:\":\"household_demographics\",\"children\":{\"Filter Operator\":{\"predicate:\":\"((hd_dep_count = 4) or (hd_vehicle_count = 3)) (type: boolean)\",\"children\":{\"Reduce Output Operator\":{\"Map-reduce partition columns:\":\"hd_demo_sk (type: int)\",\"sort order:\":\"+\",\"value expressions:\":\"hd_demo_sk (type: int), hd_dep_count (type: int), hd_vehicle_count (type: int)\",\"Statistics:\":\"Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE\",\"key expressions:\":\"hd_demo_sk (type: int)\"}},\"St
atistics:\":\"Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE\"}},\"Statistics:\":\"Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE\"}}]},\"Map 2\":{\"Map Operator Tree:\":[{\"TableScan\":{\"alias:\":\"store_sales\",\"children\":{\"Map Join Operator\":{\"keys:\":{\"1\":\"d_date_sk (type: int)\",\"0\":\"ss_sold_date_sk (type: int)\"},\"outputColumnNames:\":[\"_col0\",\"_col3\",\"_col5\",\"_col6\",\"_col8\",\"_col18\",\"_col21\",\"_col22\",\"_col25\",\"_col31\",\"_col32\"],\"children\":{\"Map Join Operator\":{\"keys:\":{\"1\":\"UDFToDouble(s_store_sk) (type: double)\",\"0\":\"UDFToDouble(_col22) (type: double)\"},\"outputColumnNames:\":[\"_col0\",\"_col3\",\"_col5\",\"_col6\",\"_col8\",\"_col18\",\"_col21\",\"_col22\",\"_col25\",\"_col31\",\"_col32\",\"_col55\",\"_col77\"],\"children\":{\"Map Join Operator\":{\"keys:\":{\"1\":\"hd_demo_sk (type: int)\",\"0\":\"_col5 (type: int)\"},\"outputColumnNames:\":[\"_col0\",\"_col22\",\"_col31
\",\"_col34\",\"_col36\",\"_col37\",\"_col39\",\"_col49\",\"_col52\",\"_col53\",\"_col56\",\"_col62\",\"_col63\",\"_col86\",\"_col89\",\"_col90\"],\"children\":{\"Map Join Operator\":{\"keys:\":{\"1\":\"ca_address_sk (type: int)\",\"0\":\"_col37 (type: int)\"},\"outputColumnNames:\":[\"_col0\",\"_col3\",\"_col4\",\"_col7\",\"_col29\",\"_col38\",\"_col41\",\"_col43\",\"_col44\",\"_col46\",\"_col56\",\"_col59\",\"_col60\",\"_col63\",\"_col69\",\"_col70\",\"_col93\",\"_col99\"],\"children\":{\"Filter Operator\":{\"predicate:\":\"((((((((_col38 = _col63) and (_col60 = _col7)) and (_col43 = _col0)) and (_col44 = _col93)) and ((_col3 = 4) or (_col4 = 3))) and (_col70) IN (6, 0)) and (_col69) IN (1999, (1999 + 1), (1999 + 2))) and (_col29) IN ('Fairview', 'Fairview', 'Fairview', 'Midway', 'Fairview')) (type: boolean)\",\"children\":{\"Select Operator\":{\"expressions:\":\"_col46 (type: int), _col41 (type: int), _col44 (type: int), _col99 (type: string), _col56 (type: decimal(7,2)), _col59
(type: decimal(7,2))\",\"outputColumnNames:\":[\"_col46\",\"_col41\",\"_col44\",\"_col99\",\"_col56\",\"_col59\"],\"children\":{\"Group By Operator\":{\"mode:\":\"hash\",\"aggregations:\":[\"sum(_col56)\",\"sum(_col59)\"],\"keys:\":\"_col46 (type: int), _col41 (type: int), _col44 (type: int), _col99 (type: string)\",\"outputColumnNames:\":[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\"],\"children\":{\"Reduce Output Operator\":{\"Map-reduce partition columns:\":\"_col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string)\",\"sort order:\":\"++++\",\"value expressions:\":\"_col4 (type: decimal(17,2)), _col5 (type: decimal(17,2))\",\"Statistics:\":\"Num rows: 32946 Data size: 43551613 Basic stats: COMPLETE Column stats: NONE\",\"key expressions:\":\"_col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string)\"}},\"Statistics:\":\"Num rows: 32946 Data size: 43551613 Basic stats: COMPLETE Column stats: NONE\"}},\"Statistics:\":\"Num rows
: 32946 Data size: 43551613 Basic stats: COMPLETE Column stats: NONE\"}},\"Statistics:\":\"Num rows: 32946 Data size: 43551613 Basic stats: COMPLETE Column stats: NONE\"}},\"Statistics:\":\"Num rows: 4217199 Data size: 5574753280 Basic stats: COMPLETE Column stats: NONE\",\"condition map:\":[{\"\":\"Inner Join 0 to 1\"}],\"condition expressions:\":{\"1\":\"{ca_address_sk} {ca_city}\",\"0\":\"{_col86} {_col89} {_col90} {_col0} {_col22} {_col31} {_col34} {_col36} {_col37} {_col39} {_col49} {_col52} {_col53} {_col56} {_col62} {_col63}\"}}},\"Statistics:\":\"Num rows: 3833817 Data size: 5067957248 Basic stats: COMPLETE Column stats: NONE\",\"condition map:\":[{\"\":\"Inner Join 0 to 1\"}],\"condition expressions:\":{\"1\":\"{hd_demo_sk} {hd_dep_count} {hd_vehicle_count}\",\"0\":\"{_col55} {_col77} {_col0} {_col3} {_col5} {_col6} {_col8} {_col18} {_col21} {_col22} {_col25} {_col31} {_col32}\"}}},\"Statistics:\":\"Num rows: 3485288 Data size: 4607233536 Basic stats: COMPLETE Column stats:
NONE\",\"condition map:\":[{\"\":\"Inner Join 0 to 1\"}],\"condition expressions:\":{\"1\":\"{s_store_sk} {s_city}\",\"0\":\"{_col0} {_col3} {_col5} {_col6} {_col8} {_col18} {_col21} {_col22} {_col25} {_col31} {_col32}\"}}},\"Statistics:\":\"Num rows: 3168444 Data size: 4188394240 Basic stats: COMPLETE Column stats: NONE\",\"condition map:\":[{\"\":\"Inner Join 0 to 1\"}],\"condition expressions:\":{\"1\":\"{d_date_sk} {d_year} {d_dow}\",\"0\":\"{ss_sold_date_sk} {ss_customer_sk} {ss_hdemo_sk} {ss_addr_sk} {ss_ticket_number} {ss_coupon_amt} {ss_net_profit} {ss_store_sk}\"}}},\"Statistics:\":\"Num rows: 2880404 Data size: 3807631184 Basic stats: COMPLETE Column stats: NONE\"}}]},\"Map 5\":{\"Execution mode:\":\"vectorized\",\"Map Operator Tree:\":[{\"TableScan\":{\"alias:\":\"customer\",\"children\":{\"Map Join Operator\":{\"keys:\":{\"1\":\"c_customer_sk (type: int)\",\"0\":\"_col1 (type: int)\"},\"outputColumnNames:\":[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\",\"
_col9\",\"_col13\",\"_col14\"],\"children\":{\"Map Join Operator\":{\"keys:\":{\"1\":\"ca_address_sk (type: int)\",\"0\":\"_col9 (type: int)\"},\"outputColumnNames:\":[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\",\"_col9\",\"_col13\",\"_col14\",\"_col25\",\"_col31\"],\"children\":{\"Filter Operator\":{\"predicate:\":\"(((_col1 = _col5) and (_col9 = _col25)) and (_col31 <> _col2)) (type: boolean)\",\"children\":{\"Select Operator\":{\"expressions:\":\"_col14 (type: string), _col13 (type: string), _col31 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: decimal(17,2)), _col4 (type: decimal(17,2))\",\"outputColumnNames:\":[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\",\"_col6\"],\"children\":{\"Reduce Output Operator\":{\"TopN Hash Memory Usage:\":\"0.04\",\"sort order:\":\"+++++\",\"value expressions:\":\"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: decimal(17,2)
), _col6 (type: decimal(17,2))\",\"Statistics:\":\"Num rows: 30250 Data size: 25902518 Basic stats: COMPLETE Column stats: NONE\",\"key expressions:\":\"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int)\"}},\"Statistics:\":\"Num rows: 30250 Data size: 25902518 Basic stats: COMPLETE Column stats: NONE\"}},\"Statistics:\":\"Num rows: 30250 Data size: 25902518 Basic stats: COMPLETE Column stats: NONE\"}},\"Statistics:\":\"Num rows: 121000 Data size: 103610072 Basic stats: COMPLETE Column stats: NONE\",\"condition map:\":[{\"\":\"Inner Join 0 to 1\"}],\"condition expressions:\":{\"1\":\"{ca_address_sk} {ca_city}\",\"0\":\"{_col0} {_col1} {_col2} {_col3} {_col4} {_col5} {_col9} {_col13} {_col14}\"}}},\"Statistics:\":\"Num rows: 110000 Data size: 94190976 Basic stats: COMPLETE Column stats: NONE\",\"condition map:\":[{\"\":\"Inner Join 0 to 1\"}],\"condition expressions:\":{\"1\":\"{c_customer_sk} {c_current_addr_sk} {c_first_name} {
c_last_name}\",\"0\":\"{_col0} {_col1} {_col2} {_col3} {_col4}\"}}},\"Statistics:\":\"Num rows: 100000 Data size: 85628164 Basic stats: COMPLETE Column stats: NONE\"}}]},\"Map 4\":{\"Map Operator Tree:\":[{\"TableScan\":{\"filterExpr:\":\"((d_dow) IN (6, 0) and (d_year) IN (1999, (1999 + 1), (1999 + 2))) (type: boolean)\",\"alias:\":\"date_dim\",\"children\":{\"Filter Operator\":{\"predicate:\":\"((d_dow) IN (6, 0) and (d_year) IN (1999, (1999 + 1), (1999 + 2))) (type: boolean)\",\"children\":{\"Reduce Output Operator\":{\"Map-reduce partition columns:\":\"d_date_sk (type: int)\",\"sort order:\":\"+\",\"value expressions:\":\"d_date_sk (type: int), d_year (type: int), d_dow (type: int)\",\"Statistics:\":\"Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE\",\"key expressions:\":\"d_date_sk (type: int)\"}},\"Statistics:\":\"Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE\"}},\"Statistics:\":\"Num rows: 73049 Data size: 81741831 B
asic stats: COMPLETE Column stats: NONE\"}}]},\"Map 7\":{\"Execution mode:\":\"vectorized\",\"Map Operator Tree:\":[{\"TableScan\":{\"alias:\":\"current_addr\",\"children\":{\"Reduce Output Operator\":{\"Map-reduce partition columns:\":\"ca_address_sk (type: int)\",\"sort order:\":\"+\",\"value expressions:\":\"ca_address_sk (type: int), ca_city (type: string)\",\"Statistics:\":\"Num rows: 50000 Data size: 55975728 Basic stats: COMPLETE Column stats: NONE\",\"key expressions:\":\"ca_address_sk (type: int)\"}},\"Statistics:\":\"Num rows: 50000 Data size: 55975728 Basic stats: COMPLETE Column stats: NONE\"}}]},\"Reducer 6\":{\"Reduce Operator Tree:\":{\"Extract\":{\"children\":{\"Limit\":{\"children\":{\"File Output Operator\":{\"Statistics:\":\"Num rows: 100 Data size: 85600 Basic stats: COMPLETE Column stats: NONE\",\"compressed:\":\"false\",\"table:\":{\"serde:\":\"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe\",\"input format:\":\"org.apache.hadoop.mapred.TextInputFormat\",\"
output format:\":\"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat\"}}},\"Statistics:\":\"Num rows: 100 Data size: 85600 Basic stats: COMPLETE Column stats: NONE\",\"Number of rows:\":\"100\"}},\"Statistics:\":\"Num rows: 30250 Data size: 25902518 Basic stats: COMPLETE Column stats: NONE\"}}},\"Map 9\":{\"Execution mode:\":\"vectorized\",\"Map Operator Tree:\":[{\"TableScan\":{\"filterExpr:\":\"(s_city) IN ('Fairview', 'Fairview', 'Fairview', 'Midway', 'Fairview') (type: boolean)\",\"alias:\":\"store\",\"children\":{\"Filter Operator\":{\"predicate:\":\"(s_city) IN ('Fairview', 'Fairview', 'Fairview', 'Midway', 'Fairview') (type: boolean)\",\"children\":{\"Reduce Output Operator\":{\"Map-reduce partition columns:\":\"UDFToDouble(s_store_sk) (type: double)\",\"sort order:\":\"+\",\"value expressions:\":\"s_store_sk (type: int), s_city (type: string)\",\"Statistics:\":\"Num rows: 6 Data size: 12816 Basic stats: COMPLETE Column stats: NONE\",\"key expressions:\":\"UDFToDoubl
e(s_store_sk) (type: double)\"}},\"Statistics:\":\"Num rows: 6 Data size: 12816 Basic stats: COMPLETE Column stats: NONE\"}},\"Statistics:\":\"Num rows: 12 Data size: 25632 Basic stats: COMPLETE Column stats: NONE\"}}]},\"Map 8\":{\"Execution mode:\":\"vectorized\",\"Map Operator Tree:\":[{\"TableScan\":{\"alias:\":\"customer_address\",\"children\":{\"Reduce Output Operator\":{\"Map-reduce partition columns:\":\"ca_address_sk (type: int)\",\"sort order:\":\"+\",\"value expressions:\":\"ca_address_sk (type: int), ca_city (type: string)\",\"Statistics:\":\"Num rows: 50000 Data size: 55975728 Basic stats: COMPLETE Column stats: NONE\",\"key expressions:\":\"ca_address_sk (type: int)\"}},\"Statistics:\":\"Num rows: 50000 Data size: 55975728 Basic stats: COMPLETE Column stats: NONE\"}}]}},\"Edges:\":{\"Reducer 3\":{\"parent\":\"Map 2\",\"type\":\"SIMPLE_EDGE\"},\"Map 2\":[{\"parent\":\"Map 1\",\"type\":\"BROADCAST_EDGE\"},{\"parent\":\"Map 4\",\"type\":\"BROADCAST_EDGE\"},{\"parent\":\"M
ap 8\",\"type\":\"BROADCAST_EDGE\"},{\"parent\":\"Map 9\",\"type\":\"BROADCAST_EDGE\"}],\"Map 5\":[{\"parent\":\"Reducer 3\",\"type\":\"BROADCAST_EDGE\"},{\"parent\":\"Map 7\",\"type\":\"BROADCAST_EDGE\"}],\"Reducer 6\":{\"parent\":\"Map 5\",\"type\":\"SIMPLE_EDGE\"}}}},\"Stage-0\":{\"Fetch Operator\":{\"limit:\":\"100\",\"Processor Tree:\":{\"ListSink\":{}}}}},\"STAGE DEPENDENCIES\":{\"Stage-1\":{\"ROOT STAGE\":\"TRUE\"},\"Stage-0\":{\"ROOT STAGE\":\"TRUE\"}}}}"
+ },
+ "entity": "hrt_qa_20140311131919_1d932567-71c2-4341-9b50-6df1f58a9114",
+ "entitytype": "HIVE_QUERY_ID"
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b76efd5b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDag-name-to-id.json
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDag-name-to-id.json b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDag-name-to-id.json
new file mode 100644
index 0000000..c04d786
--- /dev/null
+++ b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDag-name-to-id.json
@@ -0,0 +1,552 @@
+{
+ "entities": [
+ {
+ "starttime": 1394569228227,
+ "events": [
+ {
+ "timestamp": 1394569303577,
+ "eventtype": "DAG_FINISHED",
+ "eventinfo": {}
+ },
+ {
+ "timestamp": 1394569229560,
+ "eventtype": "DAG_STARTED",
+ "eventinfo": {}
+ },
+ {
+ "timestamp": 1394569229460,
+ "eventtype": "DAG_INITIALIZED",
+ "eventinfo": {}
+ },
+ {
+ "timestamp": 1394569228227,
+ "eventtype": "DAG_SUBMITTED",
+ "eventinfo": {}
+ }
+ ],
+ "otherinfo": {
+ "startTime": 1394569229560,
+ "status": "FAILED",
+ "initTime": 1394569229460,
+ "timeTaken": 74017,
+ "dagPlan": {
+ "dagName": "hrt_qa_20140311131919_1d932567-71c2-4341-9b50-6df1f58a9114:1",
+ "version": 1,
+ "vertices": [
+ {
+ "vertexName": "Reducer 6",
+ "processorClass": "org.apache.hadoop.hive.ql.exec.tez.ReduceTezProcessor",
+ "inEdgeIds": [
+ "910557679"
+ ],
+ "additionalOutputs": [
+ {
+ "name": "out_Reducer 6",
+ "class": "org.apache.tez.mapreduce.output.MROutput"
+ }
+ ]
+ },
+ {
+ "vertexName": "Map 5",
+ "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+ "inEdgeIds": [
+ "500242152",
+ "1208124175"
+ ],
+ "outEdgeIds": [
+ "910557679"
+ ],
+ "additionalInputs": [
+ {
+ "name": "customer",
+ "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+ "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+ }
+ ]
+ },
+ {
+ "vertexName": "Map 7",
+ "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+ "outEdgeIds": [
+ "500242152"
+ ],
+ "additionalInputs": [
+ {
+ "name": "current_addr",
+ "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+ "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+ }
+ ]
+ },
+ {
+ "vertexName": "Reducer 3",
+ "processorClass": "org.apache.hadoop.hive.ql.exec.tez.ReduceTezProcessor",
+ "inEdgeIds": [
+ "654658809"
+ ],
+ "outEdgeIds": [
+ "1208124175"
+ ]
+ },
+ {
+ "vertexName": "Map 2",
+ "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+ "inEdgeIds": [
+ "760294521",
+ "1982385791",
+ "813939446",
+ "1182578556"
+ ],
+ "outEdgeIds": [
+ "654658809"
+ ],
+ "additionalInputs": [
+ {
+ "name": "store_sales",
+ "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+ "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+ }
+ ]
+ },
+ {
+ "vertexName": "Map 9",
+ "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+ "outEdgeIds": [
+ "760294521"
+ ],
+ "additionalInputs": [
+ {
+ "name": "store",
+ "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+ "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+ }
+ ]
+ },
+ {
+ "vertexName": "Map 8",
+ "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+ "outEdgeIds": [
+ "1982385791"
+ ],
+ "additionalInputs": [
+ {
+ "name": "customer_address",
+ "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+ "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+ }
+ ]
+ },
+ {
+ "vertexName": "Map 4",
+ "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+ "outEdgeIds": [
+ "813939446"
+ ],
+ "additionalInputs": [
+ {
+ "name": "date_dim",
+ "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+ "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+ }
+ ]
+ },
+ {
+ "vertexName": "Map 1",
+ "processorClass": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+ "outEdgeIds": [
+ "1182578556"
+ ],
+ "additionalInputs": [
+ {
+ "name": "household_demographics",
+ "class": "org.apache.tez.mapreduce.input.MRInputLegacy",
+ "initializer": "org.apache.tez.mapreduce.common.MRInputAMSplitGenerator"
+ }
+ ]
+ }
+ ],
+ "edges": [
+ {
+ "edgeId": "760294521",
+ "inputVertexName": "Map 9",
+ "outputVertexName": "Map 2",
+ "dataMovementType": "BROADCAST",
+ "dataSourceType": "PERSISTED",
+ "schedulingType": "SEQUENTIAL",
+ "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
+ "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
+ },
+ {
+ "edgeId": "1208124175",
+ "inputVertexName": "Reducer 3",
+ "outputVertexName": "Map 5",
+ "dataMovementType": "BROADCAST",
+ "dataSourceType": "PERSISTED",
+ "schedulingType": "SEQUENTIAL",
+ "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
+ "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
+ },
+ {
+ "edgeId": "500242152",
+ "inputVertexName": "Map 7",
+ "outputVertexName": "Map 5",
+ "dataMovementType": "BROADCAST",
+ "dataSourceType": "PERSISTED",
+ "schedulingType": "SEQUENTIAL",
+ "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
+ "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
+ },
+ {
+ "edgeId": "1182578556",
+ "inputVertexName": "Map 1",
+ "outputVertexName": "Map 2",
+ "dataMovementType": "BROADCAST",
+ "dataSourceType": "PERSISTED",
+ "schedulingType": "SEQUENTIAL",
+ "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
+ "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
+ },
+ {
+ "edgeId": "910557679",
+ "inputVertexName": "Map 5",
+ "outputVertexName": "Reducer 6",
+ "dataMovementType": "SCATTER_GATHER",
+ "dataSourceType": "PERSISTED",
+ "schedulingType": "SEQUENTIAL",
+ "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileSortedOutput",
+ "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledMergedInputLegacy"
+ },
+ {
+ "edgeId": "813939446",
+ "inputVertexName": "Map 4",
+ "outputVertexName": "Map 2",
+ "dataMovementType": "BROADCAST",
+ "dataSourceType": "PERSISTED",
+ "schedulingType": "SEQUENTIAL",
+ "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
+ "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
+ },
+ {
+ "edgeId": "654658809",
+ "inputVertexName": "Map 2",
+ "outputVertexName": "Reducer 3",
+ "dataMovementType": "SCATTER_GATHER",
+ "dataSourceType": "PERSISTED",
+ "schedulingType": "SEQUENTIAL",
+ "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileSortedOutput",
+ "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledMergedInputLegacy"
+ },
+ {
+ "edgeId": "1982385791",
+ "inputVertexName": "Map 8",
+ "outputVertexName": "Map 2",
+ "dataMovementType": "BROADCAST",
+ "dataSourceType": "PERSISTED",
+ "schedulingType": "SEQUENTIAL",
+ "edgeSourceClass": "org.apache.tez.runtime.library.output.OnFileUnorderedKVOutput",
+ "edgeDestinationClass": "org.apache.tez.runtime.library.input.ShuffledUnorderedKVInput"
+ }
+ ]
+ },
+ "endTime": 1394569303577,
+ "diagnostics": "Vertex re-running, vertexName=Map 7, vertexId=vertex_1394502141829_0425_1_02\nVertex re-running, vertexName=Map 2, vertexId=vertex_1394502141829_0425_1_04\nVertex re-running, vertexName=Map 9, vertexId=vertex_1394502141829_0425_1_05\nVertex re-running, vertexName=Map 1, vertexId=vertex_1394502141829_0425_1_08\nVertex re-running, vertexName=Map 4, vertexId=vertex_1394502141829_0425_1_07\nVertex re-running, vertexName=Map 8, vertexId=vertex_1394502141829_0425_1_06\nVertex re-running, vertexName=Reducer 3, vertexId=vertex_1394502141829_0425_1_03\nVertex failed, vertexName=Map 5, vertexId=vertex_1394502141829_0425_1_01, diagnostics=[Task failed, taskId=task_1394502141829_0425_1_01_000000, diagnostics=[AttemptID:attempt_1394502141829_0425_1_01_000000_0 Info:Error: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row \n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.processRow(MapRecordProc
essor.java:205)\n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:171)\n\tat org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:152)\n\tat org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:306)\n\tat org.apache.hadoop.mapred.YarnTezDagChild$4.run(YarnTezDagChild.java:526)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:415)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548)\n\tat org.apache.hadoop.mapred.YarnTezDagChild.main(YarnTezDagChild.java:515)\nCaused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row \n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.process(VectorMapOperator.java:45)\n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.processRow(MapRecordProcessor.java:193)\n\t... 8 more\nCaused by: java.lan
g.ClassCastException: org.apache.hadoop.hive.ql.exec.vector.LongColumnVector cannot be cast to org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector\n\tat org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory$VectorExpressionWriterBytes.writeValue(VectorExpressionWriterFactory.java:268)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator.populatedCachedDistributionKeys(VectorReduceSinkOperator.java:335)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator.processOp(VectorReduceSinkOperator.java:237)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator.processOp(VectorSelectOperator.java:129)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator.processOp(VectorFilterOperator.java:102)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\
n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.flushOutput(VectorMapJoinOperator.java:249)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.internalForward(VectorMapJoinOperator.java:244)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:670)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)\n\tat org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:215)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.processOp(VectorMapJoinOperator.java:292)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.flushOutput(VectorMapJoinOperator.java:249)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.internalForward(VectorMapJoinOperator.java:244)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOne
UniqueJoinObject(CommonJoinOperator.java:670)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)\n\tat org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:215)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.processOp(VectorMapJoinOperator.java:292)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:92)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.process(VectorMapOperator.java:43)\n\t... 9 more\n\nContainer released by application, AttemptID:attempt_1394502141829_0425_1_01_000000_1 Info:Error: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row \n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.processRow(MapRecordPr
ocessor.java:205)\n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:171)\n\tat org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:152)\n\tat org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:306)\n\tat org.apache.hadoop.mapred.YarnTezDagChild$4.run(YarnTezDagChild.java:526)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:415)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548)\n\tat org.apache.hadoop.mapred.YarnTezDagChild.main(YarnTezDagChild.java:515)\nCaused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row \n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.process(VectorMapOperator.java:45)\n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.processRow(MapRecordProcessor.java:193)\n\t... 8 more\nCaused by: java.l
ang.ClassCastException: org.apache.hadoop.hive.ql.exec.vector.LongColumnVector cannot be cast to org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector\n\tat org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory$VectorExpressionWriterBytes.writeValue(VectorExpressionWriterFactory.java:268)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator.populatedCachedDistributionKeys(VectorReduceSinkOperator.java:335)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator.processOp(VectorReduceSinkOperator.java:237)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator.processOp(VectorSelectOperator.java:129)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator.processOp(VectorFilterOperator.java:102)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791
)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.flushOutput(VectorMapJoinOperator.java:249)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.internalForward(VectorMapJoinOperator.java:244)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:670)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)\n\tat org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:215)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.processOp(VectorMapJoinOperator.java:292)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.flushOutput(VectorMapJoinOperator.java:249)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.internalForward(VectorMapJoinOperator.java:244)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllO
neUniqueJoinObject(CommonJoinOperator.java:670)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)\n\tat org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:215)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.processOp(VectorMapJoinOperator.java:292)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:92)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.process(VectorMapOperator.java:43)\n\t... 9 more\n\nContainer released by application, AttemptID:attempt_1394502141829_0425_1_01_000000_2 Info:Error: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row \n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.processRow(MapRecord
Processor.java:205)\n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:171)\n\tat org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:152)\n\tat org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:306)\n\tat org.apache.hadoop.mapred.YarnTezDagChild$4.run(YarnTezDagChild.java:526)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:415)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548)\n\tat org.apache.hadoop.mapred.YarnTezDagChild.main(YarnTezDagChild.java:515)\nCaused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row \n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.process(VectorMapOperator.java:45)\n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.processRow(MapRecordProcessor.java:193)\n\t... 8 more\nCaused by: java
.lang.ClassCastException: org.apache.hadoop.hive.ql.exec.vector.LongColumnVector cannot be cast to org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector\n\tat org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory$VectorExpressionWriterBytes.writeValue(VectorExpressionWriterFactory.java:268)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator.populatedCachedDistributionKeys(VectorReduceSinkOperator.java:335)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator.processOp(VectorReduceSinkOperator.java:237)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator.processOp(VectorSelectOperator.java:129)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator.processOp(VectorFilterOperator.java:102)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:7
91)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.flushOutput(VectorMapJoinOperator.java:249)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.internalForward(VectorMapJoinOperator.java:244)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:670)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)\n\tat org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:215)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.processOp(VectorMapJoinOperator.java:292)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.flushOutput(VectorMapJoinOperator.java:249)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.internalForward(VectorMapJoinOperator.java:244)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAl
lOneUniqueJoinObject(CommonJoinOperator.java:670)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)\n\tat org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:215)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.processOp(VectorMapJoinOperator.java:292)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:92)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.process(VectorMapOperator.java:43)\n\t... 9 more\n\nContainer released by application, AttemptID:attempt_1394502141829_0425_1_01_000000_3 Info:Error: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row \n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.processRow(MapReco
rdProcessor.java:205)\n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:171)\n\tat org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:152)\n\tat org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:306)\n\tat org.apache.hadoop.mapred.YarnTezDagChild$4.run(YarnTezDagChild.java:526)\n\tat java.security.AccessController.doPrivileged(Native Method)\n\tat javax.security.auth.Subject.doAs(Subject.java:415)\n\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548)\n\tat org.apache.hadoop.mapred.YarnTezDagChild.main(YarnTezDagChild.java:515)\nCaused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row \n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.process(VectorMapOperator.java:45)\n\tat org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.processRow(MapRecordProcessor.java:193)\n\t... 8 more\nCaused by: ja
va.lang.ClassCastException: org.apache.hadoop.hive.ql.exec.vector.LongColumnVector cannot be cast to org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector\n\tat org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory$VectorExpressionWriterBytes.writeValue(VectorExpressionWriterFactory.java:268)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator.populatedCachedDistributionKeys(VectorReduceSinkOperator.java:335)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator.processOp(VectorReduceSinkOperator.java:237)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator.processOp(VectorSelectOperator.java:129)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator.processOp(VectorFilterOperator.java:102)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java
:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.flushOutput(VectorMapJoinOperator.java:249)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.internalForward(VectorMapJoinOperator.java:244)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:670)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)\n\tat org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:215)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.processOp(VectorMapJoinOperator.java:292)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.flushOutput(VectorMapJoinOperator.java:249)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.internalForward(VectorMapJoinOperator.java:244)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.gen
AllOneUniqueJoinObject(CommonJoinOperator.java:670)\n\tat org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:754)\n\tat org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:215)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator.processOp(VectorMapJoinOperator.java:292)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:92)\n\tat org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:791)\n\tat org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator.process(VectorMapOperator.java:43)\n\t... 9 more\n], Vertex failed as one or more tasks failed. failedTasks:1]\nVertex killed, vertexName=Reducer 6, vertexId=vertex_1394502141829_0425_1_00, diagnostics=[Vertex received Kill while in RUNNING state., Vertex killed as other vertex failed. failedTasks:0]\nVertex killed, vertexName=Map 2, vertexId=
vertex_1394502141829_0425_1_04, diagnostics=[Vertex received Kill while in RUNNING state., Vertex killed as other vertex failed. failedTasks:0]\nDAG failed due to vertex failure. failedVertices:1 killedVertices:2",
+ "counters": {
+ "counterGroups": [
+ {
+ "counterGroupName": "org.apache.tez.common.counters.DAGCounter",
+ "counterGroupDisplayName": "org.apache.tez.common.counters.DAGCounter",
+ "counters": [
+ {
+ "counterName": "NUM_FAILED_TASKS",
+ "counterDisplayName": "NUM_FAILED_TASKS",
+ "counterValue": 4
+ },
+ {
+ "counterName": "NUM_KILLED_TASKS",
+ "counterDisplayName": "NUM_KILLED_TASKS",
+ "counterValue": 14
+ },
+ {
+ "counterName": "TOTAL_LAUNCHED_TASKS",
+ "counterDisplayName": "TOTAL_LAUNCHED_TASKS",
+ "counterValue": 28
+ },
+ {
+ "counterName": "DATA_LOCAL_TASKS",
+ "counterDisplayName": "DATA_LOCAL_TASKS",
+ "counterValue": 9
+ }
+ ]
+ },
+ {
+ "counterGroupName": "org.apache.tez.common.counters.FileSystemCounter",
+ "counterGroupDisplayName": "File System Counters",
+ "counters": [
+ {
+ "counterName": "FILE_BYTES_READ",
+ "counterDisplayName": "FILE: BYTES_READ",
+ "counterValue": 503348
+ },
+ {
+ "counterName": "FILE_BYTES_WRITTEN",
+ "counterDisplayName": "FILE: BYTES_WRITTEN",
+ "counterValue": 4315294
+ },
+ {
+ "counterName": "FILE_READ_OPS",
+ "counterDisplayName": "FILE: READ_OPS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "FILE_LARGE_READ_OPS",
+ "counterDisplayName": "FILE: LARGE_READ_OPS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "FILE_WRITE_OPS",
+ "counterDisplayName": "FILE: WRITE_OPS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "HDFS_BYTES_READ",
+ "counterDisplayName": "HDFS: BYTES_READ",
+ "counterValue": 13041650
+ },
+ {
+ "counterName": "HDFS_BYTES_WRITTEN",
+ "counterDisplayName": "HDFS: BYTES_WRITTEN",
+ "counterValue": 0
+ },
+ {
+ "counterName": "HDFS_READ_OPS",
+ "counterDisplayName": "HDFS: READ_OPS",
+ "counterValue": 27
+ },
+ {
+ "counterName": "HDFS_LARGE_READ_OPS",
+ "counterDisplayName": "HDFS: LARGE_READ_OPS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "HDFS_WRITE_OPS",
+ "counterDisplayName": "HDFS: WRITE_OPS",
+ "counterValue": 0
+ }
+ ]
+ },
+ {
+ "counterGroupName": "org.apache.tez.common.counters.TaskCounter",
+ "counterGroupDisplayName": "org.apache.tez.common.counters.TaskCounter",
+ "counters": [
+ {
+ "counterName": "REDUCE_INPUT_GROUPS",
+ "counterDisplayName": "REDUCE_INPUT_GROUPS",
+ "counterValue": 10730
+ },
+ {
+ "counterName": "REDUCE_INPUT_RECORDS",
+ "counterDisplayName": "REDUCE_INPUT_RECORDS",
+ "counterValue": 10731
+ },
+ {
+ "counterName": "COMBINE_INPUT_RECORDS",
+ "counterDisplayName": "COMBINE_INPUT_RECORDS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "SPILLED_RECORDS",
+ "counterDisplayName": "SPILLED_RECORDS",
+ "counterValue": 17925
+ },
+ {
+ "counterName": "NUM_SHUFFLED_INPUTS",
+ "counterDisplayName": "NUM_SHUFFLED_INPUTS",
+ "counterValue": 22
+ },
+ {
+ "counterName": "NUM_FAILED_SHUFFLE_INPUTS",
+ "counterDisplayName": "NUM_FAILED_SHUFFLE_INPUTS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "MERGED_MAP_OUTPUTS",
+ "counterDisplayName": "MERGED_MAP_OUTPUTS",
+ "counterValue": 6
+ },
+ {
+ "counterName": "GC_TIME_MILLIS",
+ "counterDisplayName": "GC_TIME_MILLIS",
+ "counterValue": 2402
+ },
+ {
+ "counterName": "CPU_MILLISECONDS",
+ "counterDisplayName": "CPU_MILLISECONDS",
+ "counterValue": -33880
+ },
+ {
+ "counterName": "PHYSICAL_MEMORY_BYTES",
+ "counterDisplayName": "PHYSICAL_MEMORY_BYTES",
+ "counterValue": 3241504768
+ },
+ {
+ "counterName": "VIRTUAL_MEMORY_BYTES",
+ "counterDisplayName": "VIRTUAL_MEMORY_BYTES",
+ "counterValue": 17864921088
+ },
+ {
+ "counterName": "COMMITTED_HEAP_BYTES",
+ "counterDisplayName": "COMMITTED_HEAP_BYTES",
+ "counterValue": 2839543808
+ },
+ {
+ "counterName": "INPUT_RECORDS_PROCESSED",
+ "counterDisplayName": "INPUT_RECORDS_PROCESSED",
+ "counterValue": 2063651
+ },
+ {
+ "counterName": "OUTPUT_RECORDS",
+ "counterDisplayName": "OUTPUT_RECORDS",
+ "counterValue": 120051
+ },
+ {
+ "counterName": "OUTPUT_BYTES",
+ "counterDisplayName": "OUTPUT_BYTES",
+ "counterValue": 3571464
+ },
+ {
+ "counterName": "OUTPUT_BYTES_WITH_OVERHEAD",
+ "counterDisplayName": "OUTPUT_BYTES_WITH_OVERHEAD",
+ "counterValue": 3811586
+ },
+ {
+ "counterName": "OUTPUT_BYTES_PHYSICAL",
+ "counterDisplayName": "OUTPUT_BYTES_PHYSICAL",
+ "counterValue": 3811626
+ },
+ {
+ "counterName": "ADDITIONAL_SPILLS_BYTES_WRITTEN",
+ "counterDisplayName": "ADDITIONAL_SPILLS_BYTES_WRITTEN",
+ "counterValue": 503348
+ },
+ {
+ "counterName": "ADDITIONAL_SPILLS_BYTES_READ",
+ "counterDisplayName": "ADDITIONAL_SPILLS_BYTES_READ",
+ "counterValue": 503348
+ },
+ {
+ "counterName": "ADDITIONAL_SPILL_COUNT",
+ "counterDisplayName": "ADDITIONAL_SPILL_COUNT",
+ "counterValue": 0
+ },
+ {
+ "counterName": "SHUFFLE_BYTES",
+ "counterDisplayName": "SHUFFLE_BYTES",
+ "counterValue": 6627422
+ },
+ {
+ "counterName": "SHUFFLE_BYTES_DECOMPRESSED",
+ "counterDisplayName": "SHUFFLE_BYTES_DECOMPRESSED",
+ "counterValue": 6627334
+ },
+ {
+ "counterName": "SHUFFLE_BYTES_TO_MEM",
+ "counterDisplayName": "SHUFFLE_BYTES_TO_MEM",
+ "counterValue": 6627422
+ },
+ {
+ "counterName": "SHUFFLE_BYTES_TO_DISK",
+ "counterDisplayName": "SHUFFLE_BYTES_TO_DISK",
+ "counterValue": 0
+ },
+ {
+ "counterName": "NUM_MEM_TO_DISK_MERGES",
+ "counterDisplayName": "NUM_MEM_TO_DISK_MERGES",
+ "counterValue": 0
+ },
+ {
+ "counterName": "NUM_DISK_TO_DISK_MERGES",
+ "counterDisplayName": "NUM_DISK_TO_DISK_MERGES",
+ "counterValue": 0
+ }
+ ]
+ },
+ {
+ "counterGroupName": "Shuffle Errors",
+ "counterGroupDisplayName": "Shuffle Errors",
+ "counters": [
+ {
+ "counterName": "BAD_ID",
+ "counterDisplayName": "BAD_ID",
+ "counterValue": 0
+ },
+ {
+ "counterName": "CONNECTION",
+ "counterDisplayName": "CONNECTION",
+ "counterValue": 0
+ },
+ {
+ "counterName": "IO_ERROR",
+ "counterDisplayName": "IO_ERROR",
+ "counterValue": 0
+ },
+ {
+ "counterName": "WRONG_LENGTH",
+ "counterDisplayName": "WRONG_LENGTH",
+ "counterValue": 0
+ },
+ {
+ "counterName": "WRONG_MAP",
+ "counterDisplayName": "WRONG_MAP",
+ "counterValue": 0
+ },
+ {
+ "counterName": "WRONG_REDUCE",
+ "counterDisplayName": "WRONG_REDUCE",
+ "counterValue": 0
+ }
+ ]
+ },
+ {
+ "counterGroupName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
+ "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
+ "counters": [
+ {
+ "counterName": "FILTERED",
+ "counterDisplayName": "FILTERED",
+ "counterValue": 72735
+ },
+ {
+ "counterName": "PASSED",
+ "counterDisplayName": "PASSED",
+ "counterValue": 96225
+ }
+ ]
+ },
+ {
+ "counterGroupName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+ "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+ "counters": [
+ {
+ "counterName": "DESERIALIZE_ERRORS",
+ "counterDisplayName": "DESERIALIZE_ERRORS",
+ "counterValue": 0
+ }
+ ]
+ }
+ ]
+ }
+ },
+ "primaryfilters": {
+ "dagName": [
+ "hrt_qa_20140311131919_1d932567-71c2-4341-9b50-6df1f58a9114:1"
+ ],
+ "user": [
+ "hrt_qa"
+ ]
+ },
+ "entity": "dag_1394502141829_0425_1",
+ "entitytype": "TEZ_DAG_ID",
+ "relatedentities": {}
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b76efd5b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDag.json
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDag.json b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDag.json
new file mode 100644
index 0000000..dc60966
--- /dev/null
+++ b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDag.json
@@ -0,0 +1,21 @@
+{
+ "starttime": 1394569228227,
+ "entity": "dag_1394502141829_0425_1",
+ "entitytype": "TEZ_DAG_ID",
+ "relatedentities": {
+ "TEZ_VERTEX_ID": [
+ "vertex_1394502141829_0425_1_00",
+ "vertex_1394502141829_0425_1_01",
+ "vertex_1394502141829_0425_1_02",
+ "vertex_1394502141829_0425_1_03",
+ "vertex_1394502141829_0425_1_04",
+ "vertex_1394502141829_0425_1_05",
+ "vertex_1394502141829_0425_1_06",
+ "vertex_1394502141829_0425_1_07",
+ "vertex_1394502141829_0425_1_08"
+ ]
+ },
+ "otherinfo": {
+ "applicationId": "application_1395263571423_0014"
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b76efd5b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDagVertex.json
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDagVertex.json b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDagVertex.json
new file mode 100644
index 0000000..9946238
--- /dev/null
+++ b/contrib/views/jobs/src/main/resources/ui/app/scripts/assets/tezDagVertex.json
@@ -0,0 +1,221 @@
+{
+ "starttime": 1394569231815,
+ "otherinfo": {
+ "startTime": 1394569231819,
+ "status": "KILLED",
+ "numTasks": 6,
+ "initTime": 1394569231815,
+ "vertexName": "Map 2",
+ "timeTaken": 71735,
+ "processorClassName": "org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor",
+ "endTime": 1394569303554,
+ "initRequestedTime": 1394569230142,
+ "counters": {
+ "counterGroups": [
+ {
+ "counterGroupName": "org.apache.tez.common.counters.DAGCounter",
+ "counterGroupDisplayName": "org.apache.tez.common.counters.DAGCounter",
+ "counters": [
+ {
+ "counterName": "DATA_LOCAL_TASKS",
+ "counterDisplayName": "DATA_LOCAL_TASKS",
+ "counterValue": 4
+ }
+ ]
+ },
+ {
+ "counterGroupName": "org.apache.tez.common.counters.FileSystemCounter",
+ "counterGroupDisplayName": "File System Counters",
+ "counters": [
+ {
+ "counterName": "FILE_BYTES_READ",
+ "counterDisplayName": "FILE: BYTES_READ",
+ "counterValue": 0
+ },
+ {
+ "counterName": "FILE_BYTES_WRITTEN",
+ "counterDisplayName": "FILE: BYTES_WRITTEN",
+ "counterValue": 337559
+ },
+ {
+ "counterName": "FILE_READ_OPS",
+ "counterDisplayName": "FILE: READ_OPS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "FILE_LARGE_READ_OPS",
+ "counterDisplayName": "FILE: LARGE_READ_OPS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "FILE_WRITE_OPS",
+ "counterDisplayName": "FILE: WRITE_OPS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "HDFS_BYTES_READ",
+ "counterDisplayName": "HDFS: BYTES_READ",
+ "counterValue": 12854749
+ },
+ {
+ "counterName": "HDFS_BYTES_WRITTEN",
+ "counterDisplayName": "HDFS: BYTES_WRITTEN",
+ "counterValue": 0
+ },
+ {
+ "counterName": "HDFS_READ_OPS",
+ "counterDisplayName": "HDFS: READ_OPS",
+ "counterValue": 12
+ },
+ {
+ "counterName": "HDFS_LARGE_READ_OPS",
+ "counterDisplayName": "HDFS: LARGE_READ_OPS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "HDFS_WRITE_OPS",
+ "counterDisplayName": "HDFS: WRITE_OPS",
+ "counterValue": 0
+ }
+ ]
+ },
+ {
+ "counterGroupName": "org.apache.tez.common.counters.TaskCounter",
+ "counterGroupDisplayName": "org.apache.tez.common.counters.TaskCounter",
+ "counters": [
+ {
+ "counterName": "SPILLED_RECORDS",
+ "counterDisplayName": "SPILLED_RECORDS",
+ "counterValue": 7194
+ },
+ {
+ "counterName": "NUM_SHUFFLED_INPUTS",
+ "counterDisplayName": "NUM_SHUFFLED_INPUTS",
+ "counterValue": 16
+ },
+ {
+ "counterName": "NUM_FAILED_SHUFFLE_INPUTS",
+ "counterDisplayName": "NUM_FAILED_SHUFFLE_INPUTS",
+ "counterValue": 0
+ },
+ {
+ "counterName": "GC_TIME_MILLIS",
+ "counterDisplayName": "GC_TIME_MILLIS",
+ "counterValue": 2047
+ },
+ {
+ "counterName": "CPU_MILLISECONDS",
+ "counterDisplayName": "CPU_MILLISECONDS",
+ "counterValue": -11970
+ },
+ {
+ "counterName": "PHYSICAL_MEMORY_BYTES",
+ "counterDisplayName": "PHYSICAL_MEMORY_BYTES",
+ "counterValue": 1791279104
+ },
+ {
+ "counterName": "VIRTUAL_MEMORY_BYTES",
+ "counterDisplayName": "VIRTUAL_MEMORY_BYTES",
+ "counterValue": 7152783360
+ },
+ {
+ "counterName": "COMMITTED_HEAP_BYTES",
+ "counterDisplayName": "COMMITTED_HEAP_BYTES",
+ "counterValue": 1588592640
+ },
+ {
+ "counterName": "INPUT_RECORDS_PROCESSED",
+ "counterDisplayName": "INPUT_RECORDS_PROCESSED",
+ "counterValue": 1990495
+ },
+ {
+ "counterName": "OUTPUT_RECORDS",
+ "counterDisplayName": "OUTPUT_RECORDS",
+ "counterValue": 7194
+ },
+ {
+ "counterName": "OUTPUT_BYTES",
+ "counterDisplayName": "OUTPUT_BYTES",
+ "counterValue": 323019
+ },
+ {
+ "counterName": "OUTPUT_BYTES_WITH_OVERHEAD",
+ "counterDisplayName": "OUTPUT_BYTES_WITH_OVERHEAD",
+ "counterValue": 337415
+ },
+ {
+ "counterName": "OUTPUT_BYTES_PHYSICAL",
+ "counterDisplayName": "OUTPUT_BYTES_PHYSICAL",
+ "counterValue": 337431
+ },
+ {
+ "counterName": "ADDITIONAL_SPILLS_BYTES_WRITTEN",
+ "counterDisplayName": "ADDITIONAL_SPILLS_BYTES_WRITTEN",
+ "counterValue": 0
+ },
+ {
+ "counterName": "ADDITIONAL_SPILLS_BYTES_READ",
+ "counterDisplayName": "ADDITIONAL_SPILLS_BYTES_READ",
+ "counterValue": 0
+ },
+ {
+ "counterName": "ADDITIONAL_SPILL_COUNT",
+ "counterDisplayName": "ADDITIONAL_SPILL_COUNT",
+ "counterValue": 0
+ },
+ {
+ "counterName": "SHUFFLE_BYTES",
+ "counterDisplayName": "SHUFFLE_BYTES",
+ "counterValue": 6124044
+ },
+ {
+ "counterName": "SHUFFLE_BYTES_DECOMPRESSED",
+ "counterDisplayName": "SHUFFLE_BYTES_DECOMPRESSED",
+ "counterValue": 6123980
+ },
+ {
+ "counterName": "SHUFFLE_BYTES_TO_MEM",
+ "counterDisplayName": "SHUFFLE_BYTES_TO_MEM",
+ "counterValue": 6124044
+ },
+ {
+ "counterName": "SHUFFLE_BYTES_TO_DISK",
+ "counterDisplayName": "SHUFFLE_BYTES_TO_DISK",
+ "counterValue": 0
+ }
+ ]
+ },
+ {
+ "counterGroupName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
+ "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.FilterOperator$Counter",
+ "counters": [
+ {
+ "counterName": "FILTERED",
+ "counterDisplayName": "FILTERED",
+ "counterValue": 0
+ },
+ {
+ "counterName": "PASSED",
+ "counterDisplayName": "PASSED",
+ "counterValue": 95911
+ }
+ ]
+ },
+ {
+ "counterGroupName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+ "counterGroupDisplayName": "org.apache.hadoop.hive.ql.exec.MapOperator$Counter",
+ "counters": [
+ {
+ "counterName": "DESERIALIZE_ERRORS",
+ "counterDisplayName": "DESERIALIZE_ERRORS",
+ "counterValue": 0
+ }
+ ]
+ }
+ ]
+ },
+ "startRequestedTime": 1394569231792
+ },
+ "entity": "vertex_1394502141829_0425_1_04",
+ "entitytype": "TEZ_VERTEX_ID"
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b76efd5b/contrib/views/jobs/src/main/resources/ui/app/scripts/helpers/ajax.js
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/ui/app/scripts/helpers/ajax.js b/contrib/views/jobs/src/main/resources/ui/app/scripts/helpers/ajax.js
new file mode 100644
index 0000000..42bae63
--- /dev/null
+++ b/contrib/views/jobs/src/main/resources/ui/app/scripts/helpers/ajax.js
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Config for each ajax-request
+ *
+ * Fields example:
+ * mock - testMode url
+ * real - real url (without API prefix)
+ * type - request type (also may be defined in the format method)
+ * format - function for processing ajax params after default formatRequest. May be called with one or two parameters (data, opt). Return ajax-params object
+ * testInProduction - can this request be executed on production tests (used only in tests)
+ *
+ * @type {Object}
+ */
+var urls = {};
+/**
+ * Replace data-placeholders to its values
+ *
+ * @param {String} url
+ * @param {Object} data
+ * @return {String}
+ */
+var formatUrl = function (url, data) {
+ if (!url) return null;
+ var keys = url.match(/\{\w+\}/g);
+ keys = (keys === null) ? [] : keys;
+ if (keys) {
+ keys.forEach(function (key) {
+ var raw_key = key.substr(1, key.length - 2);
+ var replace;
+ if (!data || !data[raw_key]) {
+ replace = '';
+ }
+ else {
+ replace = data[raw_key];
+ }
+ url = url.replace(new RegExp(key, 'g'), replace);
+ });
+ }
+ return url;
+};
+
+/**
+ * this = object from config
+ * @return {Object}
+ */
+var formatRequest = function (data) {
+ var opt = {
+ type: this.type || 'GET',
+ dataType: 'json',
+ async: true,
+ headers: this.headers || {Accept: "application/json; charset=utf-8"}
+ };
+ if (App.get('testMode')) {
+ opt.url = formatUrl(this.mock ? this.mock : '', data);
+ opt.type = 'GET';
+ }
+ else {
+ var prefix = App.get('urlPrefix');
+ opt.url = prefix + formatUrl(this.real, data);
+ }
+
+ if (this.format) {
+ jQuery.extend(opt, this.format(data, opt));
+ }
+ return opt;
+};
+
+/**
+ * Wrapper for all ajax requests
+ *
+ * @type {Object}
+ */
+var ajax = Em.Object.extend({
+ /**
+ * Send ajax request
+ *
+ * @param {Object} config
+ * @return {$.ajax} jquery ajax object
+ *
+ * config fields:
+ * name - url-key in the urls-object *required*
+ * sender - object that send request (need for proper callback initialization) *required*
+ * data - object with data for url-format
+ * beforeSend - method-name for ajax beforeSend response callback
+ * success - method-name for ajax success response callback
+ * error - method-name for ajax error response callback
+ */
+ send: function (config) {
+
+ Ember.assert('Ajax sender should be defined!', config.sender);
+ Ember.assert('Invalid config.name provided - ' + config.name, urls[config.name]);
+
+ var opt = {},
+ params = {};
+
+ if (config.data) {
+ jQuery.extend(params, config.data);
+ }
+
+ opt = formatRequest.call(urls[config.name], params);
+ opt.context = this;
+
+ // object sender should be provided for processing beforeSend, success, error and complete responses
+ opt.beforeSend = function (xhr) {
+ if (config.beforeSend) {
+ config.sender[config.beforeSend](opt, xhr, params);
+ }
+ };
+
+ opt.success = function (data) {
+ console.log("TRACE: The url is: " + opt.url);
+ if (config.success) {
+ config.sender[config.success](data, opt, params);
+ }
+ };
+
+ opt.error = function (request, ajaxOptions, error) {
+ if (config.error) {
+ config.sender[config.error](request, ajaxOptions, error, opt, params);
+ }
+ };
+
+ opt.complete = function (xhr, status) {
+ if (config.complete) {
+ config.sender[config.complete](xhr, status);
+ }
+ };
+
+ return $.ajax(opt);
+ }
+
+});
+
+App.ajax = ajax.create({});
http://git-wip-us.apache.org/repos/asf/ambari/blob/b76efd5b/contrib/views/jobs/src/main/resources/ui/app/scripts/helpers/date.js
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/ui/app/scripts/helpers/date.js b/contrib/views/jobs/src/main/resources/ui/app/scripts/helpers/date.js
index 66a5fb1..b8bb058 100644
--- a/contrib/views/jobs/src/main/resources/ui/app/scripts/helpers/date.js
+++ b/contrib/views/jobs/src/main/resources/ui/app/scripts/helpers/date.js
@@ -138,7 +138,8 @@ App.Helpers.date = {
return '' + this.timingFormat(endTimestamp - startTimestamp, 1); //lasted for xx secs
} else {
// still running, duration till now
- var time = (App.dateTime() - startTimestamp) < 0 ? 0 : (App.dateTime() - startTimestamp);
+ var t = new Date().getTime(),
+ time = (t - startTimestamp) < 0 ? 0 : (t - startTimestamp);
durationSummary = '' + this.timingFormat(time, 1);
}
return durationSummary;
@@ -207,7 +208,7 @@ App.Helpers.date = {
var duration = 0;
if (startTime && startTime > 0) {
if (!endTime || endTime < 1) {
- endTime = App.dateTime();
+ endTime = new Date().getTime();
}
duration = endTime - startTime;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b76efd5b/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/jobs/hive_job_mapper.js
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/jobs/hive_job_mapper.js b/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/jobs/hive_job_mapper.js
new file mode 100644
index 0000000..ad65902
--- /dev/null
+++ b/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/jobs/hive_job_mapper.js
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * Maps a Hive job response from server into an existing Hive Job.
+ *
+ * This will only update an existing Hive Job and not create a new one. It will
+ * populate all fields (stages, Tez DAG, etc.) except runtime information like
+ * <ul>
+ * <li>tezDag.id
+ * <li>tezDag.vertices.state
+ * <li>tezDag.vertices.startTime
+ * <li>tezDag.vertices.endTime
+ * <li>tezDag.vertices.tasksCount
+ * <li>tezDag.vertices.file*
+ * <li>tezDag.vertices.hdfs*
+ * <li>tezDag.vertices.recordReadCount
+ * <li>tezDag.vertices.recordWriteCount
+ * </ul>
+ */
+App.hiveJobMapper = App.QuickDataMapper.create({
+
+ model : App.HiveJob,
+ map : function(json) {
+ var model = this.get('model'),
+ hiveJob = {};
+ if (json && json.entity) {
+ hiveJob.id = json.entity;
+ hiveJob.name = hiveJob.id;
+ hiveJob.startTime = json.starttime;
+ if (typeof(json.endtime) == "undefined") {
+ var i = 0;
+ while (typeof(hiveJob.endTime) == "undefined" && json.events && json.events[i]) {
+ if (json.events[i].eventtype == 'QUERY_COMPLETED') {
+ hiveJob.endTime = json.events[i].timestamp;
+ }
+ i++;
+ }
+ }
+ else {
+ hiveJob.endTime = json.endtime;
+ }
+ json.otherinfo.query = $.parseJSON(json.otherinfo.query);
+ if (json.otherinfo.query && json.otherinfo.query.queryText) {
+ hiveJob.query_text = json.otherinfo.query.queryText;
+ }
+ hiveJob.stages = [];
+ var stagePlans = json.otherinfo.query.queryPlan["STAGE PLANS"];
+ for ( var stage in stagePlans) {
+ var stageValue = stagePlans[stage];
+ var stageItem = {};
+ stageItem.id = stage;
+ stageItem.description = '. ';
+ for (var item in stageValue) {
+ stageItem.description += item;
+ }
+ hiveJob.stages.push(stageItem);
+ if (stageValue.Tez != null && hiveJob.tezDag == null) {
+ var dagName = stageValue.Tez['DagName:'];
+ // Vertices
+ var vertices = [];
+ var vertexIds = [];
+ var vertexIdMap = {};
+ for ( var vertexName in stageValue.Tez["Vertices:"]) {
+ var vertex = stageValue.Tez["Vertices:"][vertexName];
+ var vertexObj = {
+ id : dagName + "/" + vertexName,
+ name : vertexName,
+ incoming_edges : [],
+ outgoing_edges : []
+ };
+ vertexIds.push(vertexObj.id);
+ var operatorExtractor = function(obj) {
+ var ops = [];
+ if ($.isArray(obj)) {
+ obj.forEach(function(o) {
+ ops = ops.concat(operatorExtractor(o));
+ });
+ }
+ else {
+ for ( var key in obj) {
+ ops.push(key);
+ if (obj[key].children != null) {
+ ops = ops.concat(operatorExtractor(obj[key].children));
+ }
+ }
+ }
+ return ops;
+ };
+ if (vertex["Map Operator Tree:"] != null) {
+ vertexObj.type = App.TezDagVertexType.MAP;
+ vertexObj.operations = operatorExtractor(vertex["Map Operator Tree:"]);
+ vertexObj.operation_plan = JSON.stringify(vertex["Map Operator Tree:"], undefined, " ");
+ }
+ else
+ if (vertex["Reduce Operator Tree:"] != null) {
+ vertexObj.type = App.TezDagVertexType.REDUCE;
+ vertexObj.operations = operatorExtractor(vertex["Reduce Operator Tree:"]);
+ vertexObj.operation_plan = JSON.stringify(vertex["Reduce Operator Tree:"], undefined, " ");
+ }
+ else
+ if (vertex["Vertex:"] != null && vertexName==vertex['Vertex:']) {
+ vertexObj.type = App.TezDagVertexType.UNION;
+ }
+ vertexIdMap[vertexObj.id] = vertexObj;
+ vertices.push(vertexObj);
+ }
+ // Edges
+ var edges = [];
+ var edgeIds = [];
+ for ( var childVertex in stageValue.Tez["Edges:"]) {
+ var childVertices = stageValue.Tez["Edges:"][childVertex];
+ if (!$.isArray(childVertices)) {
+ // Single edge given as object instead of array
+ childVertices = [ childVertices ];
+ }
+ childVertices.forEach(function(e) {
+ var parentVertex = e.parent;
+ if (e.type == 'CONTAINS') {
+ var parentVertexNode = vertexIdMap[dagName + "/" + parentVertex];
+ if (parentVertexNode != null && parentVertexNode.type == App.TezDagVertexType.UNION) {
+ // We flip the edges for Union vertices
+ var tmp = childVertex;
+ childVertex = parentVertex;
+ parentVertex = tmp;
+ }
+ }
+ var edgeObj = {
+ id : dagName + "/" + parentVertex + "-" + childVertex,
+ from_vertex_id : dagName + "/" + parentVertex,
+ to_vertex_id : dagName + "/" + childVertex
+ };
+ vertexIdMap[edgeObj.from_vertex_id].outgoing_edges.push(edgeObj.id);
+ vertexIdMap[edgeObj.to_vertex_id].incoming_edges.push(edgeObj.id);
+ edgeIds.push(edgeObj.id);
+ switch (e.type) {
+ case "BROADCAST_EDGE":
+ edgeObj.edge_type = App.TezDagEdgeType.BROADCAST;
+ break;
+ case "SIMPLE_EDGE":
+ edgeObj.edge_type = App.TezDagEdgeType.SCATTER_GATHER;
+ break;
+ case "CONTAINS":
+ edgeObj.edge_type = App.TezDagEdgeType.CONTAINS;
+ break;
+ default:
+ break;
+ }
+ edges.push(edgeObj);
+ });
+ }
+ // Create records
+ var tezDag = {
+ id : dagName,
+ name : dagName,
+ stage : stage,
+ vertices : vertexIds,
+ edges : edgeIds
+ };
+ // Once the DAG is loaded, we do not need to
+ // reload as the structure does not change. Reloading
+ // here causes missing data (got from other calls)
+ // to propagate into UI - causing flashing.
+ var newVertices = [];
+ var newEdges = [];
+ vertices.forEach(function(v) {
+ var vertexRecord = App.TezDagVertex.find(v.id);
+ if (!vertexRecord.get('isLoaded')) {
+ newVertices.push(v);
+ }
+ });
+ edges.forEach(function(e) {
+ var edgeRecord = App.TezDagEdge.find(e.id);
+ if (!edgeRecord.get('isLoaded')) {
+ newEdges.push(e);
+ }
+ });
+ App.store.loadMany(App.TezDagVertex, newVertices);
+ App.store.loadMany(App.TezDagEdge, newEdges);
+ var dagRecord = App.TezDag.find(tezDag.id);
+ if (!dagRecord.get('isLoaded')) {
+ App.store.load(App.TezDag, tezDag);
+ }
+ hiveJob.tezDag = tezDag.id;
+ }
+ }
+ if(App.HiveJob.find().get('content').length == 0){
+ App.store.load(model, hiveJob);
+ }
+ var hiveJobRecord = App.HiveJob.find(hiveJob.id);
+ if (hiveJobRecord != null) {
+ hiveJobRecord.set('stages', hiveJob.stages.sortProperty('id'));
+ hiveJobRecord.set('startTime', hiveJob.startTime);
+ hiveJobRecord.set('endTime', hiveJob.endTime);
+ if (hiveJob.tezDag != null) {
+ // Some hive queries dont use Tez
+ hiveJobRecord.set('tezDag', App.TezDag.find(hiveJob.tezDag));
+ }
+ }
+ }
+ },
+ config : {}
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/b76efd5b/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/jobs/hive_jobs_mapper.js
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/jobs/hive_jobs_mapper.js b/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/jobs/hive_jobs_mapper.js
new file mode 100644
index 0000000..9a1fc07
--- /dev/null
+++ b/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/jobs/hive_jobs_mapper.js
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+App.hiveJobsMapper = App.QuickDataMapper.create({
+
+ model: App.HiveJob,
+
+ map: function (json) {
+
+ var model = this.get('model'),
+ hiveJobs = [];
+ if (json) {
+ if (!json.entities) {
+ json.entities = [];
+ if (json.entity) {
+ json.entities = [json];
+ }
+ }
+ var currentEntityMap = {};
+ json.entities.forEach(function (entity) {
+ currentEntityMap[entity.entity] = entity.entity;
+ var hiveJob = {
+ id: entity.entity,
+ name: entity.entity,
+ user: entity.primaryfilters.user
+ };
+ hiveJob.has_tez_dag = false;
+ hiveJob.query_text = '';
+ if (entity.otherinfo && entity.otherinfo.query) {
+ // Explicit false match needed for when failure hook not set
+ hiveJob.failed = entity.otherinfo.status === false;
+ hiveJob.has_tez_dag = entity.otherinfo.query.match("\"Tez\".*\"DagName:\"");
+ var queryJson = $.parseJSON(entity.otherinfo.query);
+ if (queryJson && queryJson.queryText) {
+ hiveJob.query_text = queryJson.queryText;
+ }
+ }
+ if (entity.events != null) {
+ entity.events.forEach(function (event) {
+ switch (event.eventtype) {
+ case "QUERY_SUBMITTED":
+ hiveJob.start_time = event.timestamp;
+ break;
+ case "QUERY_COMPLETED":
+ hiveJob.end_time = event.timestamp;
+ break;
+ default:
+ break;
+ }
+ });
+ }
+ if (!hiveJob.start_time && entity.starttime > 0) {
+ hiveJob.start_time = entity.starttime;
+ }
+ if (!hiveJob.end_time && entity.endtime > 0) {
+ hiveJob.end_time = entity.endtime;
+ }
+ hiveJobs.push(hiveJob);
+ hiveJob = null;
+ entity = null;
+ });
+
+ /*if(hiveJobs.length > App.router.get('mainJobsController.filterObject.jobsLimit')) {
+ var lastJob = hiveJobs.pop();
+ if(App.router.get('mainJobsController.navIDs.nextID') != lastJob.id) {
+ App.router.set('mainJobsController.navIDs.nextID', lastJob.id);
+ }
+ currentEntityMap[lastJob.id] = null;
+ }*/
+
+ // Delete IDs not seen from server
+ /*var hiveJobsModel = model.find().toArray();
+ hiveJobsModel.forEach(function(job) {
+ if (job && !currentEntityMap[job.get('id')]) {
+ this.deleteRecord(job);
+ }
+ }, this);*/
+ }
+ App.HiveJob.store.pushMany('hiveJob', hiveJobs);
+ },
+ config: {}
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/b76efd5b/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/server_data_mapper.js
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/server_data_mapper.js b/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/server_data_mapper.js
new file mode 100644
index 0000000..02f17ff
--- /dev/null
+++ b/contrib/views/jobs/src/main/resources/ui/app/scripts/mappers/server_data_mapper.js
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App.ServerDataMapper = Em.Object.extend({
+ jsonKey: false,
+ map: function (json) {
+ if (json) {
+ var model = this.get('model');
+ var jsonKey = this.get('jsonKey');
+
+ if (jsonKey && json[jsonKey]) {
+ json = json[jsonKey];
+ }
+
+ $.each(json, function (field, value) {
+ model.set(field, value);
+ })
+ }
+ }
+});
+
+
+App.QuickDataMapper = App.ServerDataMapper.extend({
+ config: {},
+ model: null,
+ map: Em.K,
+
+ parseIt: function (data, config) {
+ var result = {};
+ for ( var i in config) {
+ if (i.substr(0, 1) === '$') {
+ i = i.substr(1, i.length);
+ result[i] = config['$' + i];
+ } else {
+ var isSpecial = false;
+ if (i.substr(-5) == '_type') {
+ var prefix = i.substr(0, i.length - 5);
+ isSpecial = config[prefix + '_key'] != null;
+ } else if (i.substr(-4) == '_key') {
+ var prefix = i.substr(0, i.length - 4);
+ isSpecial = config[prefix + '_type'] != null;
+ }
+ if (!isSpecial && typeof config[i] == 'string') {
+ result[i] = this.getJsonProperty(data, config[i]);
+ } else if (typeof config[i] == 'object') {
+ result[i] = [];
+ var _data = this.getJsonProperty(data, config[i + '_key']);
+ var _type = config[i + '_type'];
+ var l = _data.length;
+ for ( var index = 0; index < l; index++) {
+ if (_type == 'array') {
+ result[i].push(this.getJsonProperty(_data[index], config[i].item));
+ } else {
+ result[i].push(this.parseIt(_data[index], config[i]));
+ }
+ }
+ if(_type == 'array'){
+ result[i] = result[i].sort();
+ }
+ }
+ }
+ }
+ return result;
+ },
+
+ getJsonProperty: function (json, path) {
+ var pathArr = path.split('.');
+ var current = json;
+ pathArr = this.filterDotted(pathArr);
+ while (pathArr.length && current) {
+ if (pathArr[0].substr(-1) == ']') {
+ var index = parseInt(pathArr[0].substr(-2, 1));
+ var attr = pathArr[0].substr(0, pathArr[0].length - 3);
+ if (attr in current) {
+ current = current[attr][index];
+ }
+ } else {
+ current = current[pathArr[0]];
+ }
+ pathArr.splice(0, 1);
+ }
+ return current;
+ },
+
+ filterDotted: function(arr) {
+ var buffer = [];
+ var dottedBuffer = [];
+ var dotted = false;
+ arr.forEach(function(item) {
+ if(/\['|\["/.test(item)) {
+ dottedBuffer.push(item.substr(2, item.length));
+ dotted = true;
+ } else if (dotted && !/\]'|"\]/.test(item)) {
+ dottedBuffer.push(item);
+ } else if (/']|"]/.test(item)) {
+ dottedBuffer.push(item.substr(0, item.length - 2));
+ buffer.push(dottedBuffer.join('.'));
+ dotted = false;
+ dottedBuffer = [];
+ } else {
+ buffer.push(item);
+ }
+ });
+
+ return buffer;
+ },
+
+ /**
+ * properly delete record from model
+ * @param item
+ */
+ deleteRecord: function (item) {
+ item.deleteRecord();
+ App.store.commit();
+ item.get('stateManager').transitionTo('loading');
+ console.log('Record with id:' + item.get('id') + ' was deleted from model');
+ }
+});