You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@chukwa.apache.org by ey...@apache.org on 2015/06/25 22:48:53 UTC
[22/22] chukwa git commit: CHUKWA-769. Updated Chukwa to current
HBase API. (Eric Yang)
CHUKWA-769. Updated Chukwa to current HBase API. (Eric Yang)
Project: http://git-wip-us.apache.org/repos/asf/chukwa/repo
Commit: http://git-wip-us.apache.org/repos/asf/chukwa/commit/384d3449
Tree: http://git-wip-us.apache.org/repos/asf/chukwa/tree/384d3449
Diff: http://git-wip-us.apache.org/repos/asf/chukwa/diff/384d3449
Branch: refs/heads/master
Commit: 384d3449ce18694ac58ab77754f18ec87492ae00
Parents: f3dfc94
Author: Eric Yang <ey...@apache.org>
Authored: Thu Jun 25 12:33:44 2015 -0700
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Jun 25 12:33:44 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../writer/hbase/HBaseWriter.java | 3 --
.../datacollection/writer/hbase/Reporter.java | 53 ++++++++++++--------
.../chukwa/datastore/ChukwaHBaseStore.java | 51 +++++++++----------
.../hadoop/chukwa/hicc/HiccWebServer.java | 2 -
.../chukwa/hicc/rest/MetricsController.java | 42 ++++++++--------
6 files changed, 79 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 633a7c9..26999a0 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -18,6 +18,8 @@ Trunk (unreleased changes)
IMPROVEMENTS
+ CHUKWA-769. Updated Chukwa to current HBase API. (Eric Yang)
+
CHUKWA-767. Implemented low pass filter for Charting REST API. (Eric Yang)
CHUKWA-765. Minor stylesheets clean up. (Eric Yang)
http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
index 0718feb..621f22e 100644
--- a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
+++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
@@ -40,9 +40,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.log4j.Logger;
http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java
index e6e1f1b..30442e2 100644
--- a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java
+++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/Reporter.java
@@ -19,15 +19,20 @@
package org.apache.hadoop.chukwa.datacollection.writer.hbase;
import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Type;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hbase.client.Put;
-import org.json.simple.JSONObject;
import org.mortbay.log.Log;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
public class Reporter {
private ArrayList<Put> meta = new ArrayList<Put>();
private MessageDigest md5 = null;
@@ -38,33 +43,38 @@ public class Reporter {
public void putSource(String type, String source) {
byte[] value = getHash(source);
- JSONObject json = new JSONObject();
+ String buffer;
try {
- json.put("sig", new String(value, "UTF-8"));
- json.put("type", "source");
- } catch (UnsupportedEncodingException e) {
+ Type metaType = new TypeToken<Map<String, String>>(){}.getType();
+ Map<String, String> meta = new HashMap<String, String>();
+ meta.put("sig", new String(value, "UTF-8"));
+ meta.put("type", "source");
+ Gson gson = new Gson();
+ buffer = gson.toJson(meta, metaType);
+ put(type.getBytes(), source.getBytes(), buffer.toString().getBytes());
+ } catch (Exception e) {
Log.warn("Error encoding metadata.");
Log.warn(e);
}
- put(type.getBytes(), source.getBytes(), json.toString().getBytes());
-
}
public void putMetric(String type, String metric) {
String buf = new StringBuilder(type).append(".").append(metric).toString();
byte[] pk = getHash(buf);
-
- JSONObject json = new JSONObject();
+ String buffer;
try {
- json.put("sig", new String(pk, "UTF-8"));
- json.put("type", "metric");
- } catch (UnsupportedEncodingException e) {
+ Type metaType = new TypeToken<Map<String, String>>(){}.getType();
+ Map<String, String> meta = new HashMap<String, String>();
+ meta.put("sig", new String(pk, "UTF-8"));
+ meta.put("type", "metric");
+ Gson gson = new Gson();
+ buffer = gson.toJson(meta, metaType);
+ put(type.getBytes(), metric.getBytes(), buffer.toString().getBytes());
+ } catch (Exception e) {
Log.warn("Error encoding metadata.");
Log.warn(e);
}
- put(type.getBytes(), metric.getBytes(), json.toString().getBytes());
-
}
public void put(String key, String source, String info) {
@@ -73,7 +83,7 @@ public class Reporter {
public void put(byte[] key, byte[] source, byte[] info) {
Put put = new Put(key);
- put.add("k".getBytes(), source, info);
+ put.addColumn("k".getBytes(), source, info);
meta.add(put);
}
@@ -93,16 +103,19 @@ public class Reporter {
public void putClusterName(String type, String clusterName) {
byte[] value = getHash(clusterName);
- JSONObject json = new JSONObject();
-
+ String buffer;
try {
- json.put("sig", new String(value, "UTF-8"));
- json.put("type", "cluster");
+ Type metaType = new TypeToken<Map<String, String>>(){}.getType();
+ Map<String, String> meta = new HashMap<String, String>();
+ meta.put("sig", new String(value, "UTF-8"));
+ meta.put("type", "cluster");
+ Gson gson = new Gson();
+ buffer = gson.toJson(meta, metaType);
+ put(type.getBytes(), clusterName.getBytes(), buffer.toString().getBytes());
} catch (UnsupportedEncodingException e) {
Log.warn("Error encoding metadata.");
Log.warn(e);
}
- put(type.getBytes(), clusterName.getBytes(), json.toString().getBytes());
}
}
http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java b/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
index 2bef452..cd8a15f 100644
--- a/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
+++ b/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.chukwa.util.ExceptionUtil;
import org.apache.hadoop.chukwa.util.HBaseUtil;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
@@ -63,7 +62,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
-import org.mortbay.log.Log;
import com.google.gson.Gson;
@@ -72,7 +70,7 @@ public class ChukwaHBaseStore {
static int MINUTES_IN_HOUR = 60;
static double RESOLUTION = 360;
static int MINUTE = 60000; //60 milliseconds
- static int SECOND = 1000;
+ final static int SECOND = (int) TimeUnit.SECONDS.toMillis(1);
static byte[] COLUMN_FAMILY = "t".getBytes();
static byte[] ANNOTATION_FAMILY = "a".getBytes();
@@ -169,11 +167,11 @@ public class ChukwaHBaseStore {
while (it.hasNext()) {
Result result = it.next();
- for (KeyValue kv : result.raw()) {
- byte[] key = kv.getQualifier();
+ for (Cell kv : result.rawCells()) {
+ byte[] key = CellUtil.cloneQualifier(kv);
long timestamp = ByteBuffer.wrap(key).getLong();
double value = Double
- .parseDouble(new String(kv.getValue(), "UTF-8"));
+ .parseDouble(new String(CellUtil.cloneValue(kv), "UTF-8"));
series.add(timestamp, value);
}
}
@@ -195,10 +193,10 @@ public class ChukwaHBaseStore {
Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
Get get = new Get(metricGroup.getBytes());
Result result = table.get(get);
- for (KeyValue kv : result.raw()) {
- JSONObject json = (JSONObject) JSONValue.parse(new String(kv.getValue(), "UTF-8"));
+ for (Cell kv : result.rawCells()) {
+ JSONObject json = (JSONObject) JSONValue.parse(new String(CellUtil.cloneValue(kv), "UTF-8"));
if (json.get("type").equals("metric")) {
- familyNames.add(new String(kv.getQualifier(), "UTF-8"));
+ familyNames.add(new String(CellUtil.cloneQualifier(kv), "UTF-8"));
}
}
table.close();
@@ -243,9 +241,9 @@ public class ChukwaHBaseStore {
while (it.hasNext()) {
Result result = it.next();
for (Cell cell : result.rawCells()) {
- JSONObject json = (JSONObject) JSONValue.parse(new String(cell.getValue(), "UTF-8"));
+ JSONObject json = (JSONObject) JSONValue.parse(new String(CellUtil.cloneValue(cell), "UTF-8"));
if (json!=null && json.get("type")!=null && json.get("type").equals("source")) {
- pk.add(new String(cell.getQualifier(), "UTF-8"));
+ pk.add(new String(CellUtil.cloneQualifier(cell), "UTF-8"));
}
}
}
@@ -260,7 +258,6 @@ public class ChukwaHBaseStore {
public static Heatmap getHeatmap(String metricGroup, String metric,
long startTime, long endTime, double max, double scale, int width, int height) {
- final long SECONDS = TimeUnit.SECONDS.toMillis(1);
Heatmap heatmap = new Heatmap();
Set<String> sources = getSourceNames(metricGroup);
Set<String> metrics = getMetricNames(metricGroup);
@@ -298,7 +295,7 @@ public class ChukwaHBaseStore {
for (Result result : rs) {
for(Cell cell : result.rawCells()) {
byte[] dest = new byte[5];
- System.arraycopy(cell.getRow(), 3, dest, 0, 5);
+ System.arraycopy(CellUtil.cloneRow(cell), 3, dest, 0, 5);
String source = new String(dest);
long time = cell.getTimestamp();
// Time display in x axis
@@ -358,9 +355,9 @@ public class ChukwaHBaseStore {
while (it.hasNext()) {
Result result = it.next();
for (Cell cell : result.rawCells()) {
- JSONObject json = (JSONObject) JSONValue.parse(new String(cell.getValue(), "UTF-8"));
+ JSONObject json = (JSONObject) JSONValue.parse(new String(CellUtil.cloneValue(cell), "UTF-8"));
if (json.get("type").equals("cluster")) {
- clusters.add(new String(cell.getQualifier(), "UTF-8"));
+ clusters.add(new String(CellUtil.cloneQualifier(cell), "UTF-8"));
}
}
}
@@ -411,7 +408,7 @@ public class ChukwaHBaseStore {
Put put = new Put(CHART_TYPE);
Gson gson = new Gson();
String buffer = gson.toJson(chart);
- put.add(CHART_FAMILY, id.getBytes(), buffer.getBytes());
+ put.addColumn(CHART_FAMILY, id.getBytes(), buffer.getBytes());
table.put(put);
table.close();
} catch (Exception e) {
@@ -472,7 +469,7 @@ public class ChukwaHBaseStore {
Put put = new Put(CHART_TYPE);
Gson gson = new Gson();
String buffer = gson.toJson(chart);
- put.add(CHART_FAMILY, id.getBytes(), buffer.getBytes());
+ put.addColumn(CHART_FAMILY, id.getBytes(), buffer.getBytes());
table.put(put);
table.close();
} catch (Exception e) {
@@ -549,10 +546,10 @@ public class ChukwaHBaseStore {
while (it.hasNext()) {
Result result = it.next();
- for (KeyValue kv : result.raw()) {
- byte[] key = kv.getQualifier();
+ for (Cell kv : result.rawCells()) {
+ byte[] key = CellUtil.cloneQualifier(kv);
long timestamp = ByteBuffer.wrap(key).getLong();
- double value = Double.parseDouble(new String(kv.getValue(),
+ double value = Double.parseDouble(new String(CellUtil.cloneValue(kv),
"UTF-8"));
if(initial==0) {
filteredValue = value;
@@ -617,7 +614,7 @@ public class ChukwaHBaseStore {
int c = 0;
while(it.hasNext()) {
Result result = it.next();
- for(KeyValue kv : result.raw()) {
+ for(Cell kv : result.rawCells()) {
if(c > limit) {
break;
}
@@ -625,7 +622,7 @@ public class ChukwaHBaseStore {
continue;
}
Gson gson = new Gson();
- Widget widget = gson.fromJson(new String(kv.getValue(), "UTF-8"), Widget.class);
+ Widget widget = gson.fromJson(new String(CellUtil.cloneValue(kv), "UTF-8"), Widget.class);
list.add(widget);
c++;
}
@@ -659,9 +656,9 @@ public class ChukwaHBaseStore {
Iterator<Result> it = rs.iterator();
while(it.hasNext()) {
Result result = it.next();
- for(KeyValue kv : result.raw()) {
+ for(Cell kv : result.rawCells()) {
Gson gson = new Gson();
- Widget widget = gson.fromJson(new String(kv.getValue(), "UTF-8"), Widget.class);
+ Widget widget = gson.fromJson(new String(CellUtil.cloneValue(kv), "UTF-8"), Widget.class);
list.add(widget);
}
}
@@ -719,7 +716,7 @@ public class ChukwaHBaseStore {
Put put = new Put(WIDGET_TYPE);
Gson gson = new Gson();
String buffer = gson.toJson(widget);
- put.add(COMMON_FAMILY, widget.getTitle().getBytes(), buffer.getBytes());
+ put.addColumn(COMMON_FAMILY, widget.getTitle().getBytes(), buffer.getBytes());
table.put(put);
created = true;
}
@@ -749,7 +746,7 @@ public class ChukwaHBaseStore {
Put put = new Put(WIDGET_TYPE);
Gson gson = new Gson();
String buffer = gson.toJson(widget);
- put.add(COMMON_FAMILY, title.getBytes(), buffer.getBytes());
+ put.addColumn(COMMON_FAMILY, title.getBytes(), buffer.getBytes());
table.put(put);
table.close();
result = true;
@@ -947,7 +944,7 @@ public class ChukwaHBaseStore {
Put put = new Put(DASHBOARD_TYPE);
Gson gson = new Gson();
String buffer = gson.toJson(dash);
- put.add(COMMON_FAMILY, key.getBytes(), buffer.getBytes());
+ put.addColumn(COMMON_FAMILY, key.getBytes(), buffer.getBytes());
table.put(put);
table.close();
result = true;
http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java b/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java
index 073fc35..2d84c09 100644
--- a/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java
+++ b/src/main/java/org/apache/hadoop/chukwa/hicc/HiccWebServer.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
@@ -32,7 +31,6 @@ import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
-import org.apache.hadoop.chukwa.datastore.ChukwaHBaseStore;
import org.apache.hadoop.chukwa.util.ExceptionUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
http://git-wip-us.apache.org/repos/asf/chukwa/blob/384d3449/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java b/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java
index 9730b43..5130523 100644
--- a/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java
+++ b/src/main/java/org/apache/hadoop/chukwa/hicc/rest/MetricsController.java
@@ -17,8 +17,11 @@
*/
package org.apache.hadoop.chukwa.hicc.rest;
+import java.lang.reflect.Type;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Set;
import javax.servlet.http.HttpServletRequest;
@@ -35,9 +38,9 @@ import javax.ws.rs.core.Response;
import org.apache.hadoop.chukwa.datastore.ChukwaHBaseStore;
import org.apache.hadoop.chukwa.hicc.TimeHandler;
import org.apache.hadoop.chukwa.hicc.bean.Series;
-import org.json.simple.JSONArray;
import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
@Path("/metrics")
public class MetricsController {
@@ -95,15 +98,16 @@ public class MetricsController {
if(skey!=null) {
HttpSession session = request.getSession();
String[] sourcekeys = (session.getAttribute(skey).toString()).split(",");
- JSONArray seriesList = new JSONArray();
+ Type seriesListType =new TypeToken<ArrayList<Series>>(){}.getType();
+ ArrayList<Series> seriesList = new ArrayList<Series>();
for(String source : sourcekeys) {
- if (source == null || source.equals("")) {
- continue;
- }
+ if (source == null || source.equals("")) {
+ continue;
+ }
Series output = ChukwaHBaseStore.getSeries(metricGroup, metric, source, startTime, endTime);
- seriesList.add(output.toJSONObject());
+ seriesList.add(output);
}
- buffer = seriesList.toString();
+ buffer = new Gson().toJson(seriesList, seriesListType);
} else {
throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST)
.entity("No session attribute key defined.").build());
@@ -120,11 +124,9 @@ public class MetricsController {
@Produces("application/json")
public String getTables() {
Set<String> metricGroups = ChukwaHBaseStore.getMetricGroups();
- JSONArray groups = new JSONArray();
- for(String metric : metricGroups) {
- groups.add(metric);
- }
- return groups.toString();
+ Type metricGroupsType = new TypeToken<List<String>>(){}.getType();
+ String groups = new Gson().toJson(metricGroups, metricGroupsType);
+ return groups;
}
@GET
@@ -132,11 +134,9 @@ public class MetricsController {
@Produces("application/json")
public String getMetrics(@PathParam("metricGroup") String metricGroup) {
Set<String> metricNames = ChukwaHBaseStore.getMetricNames(metricGroup);
- JSONArray metrics = new JSONArray();
- for(String metric : metricNames) {
- metrics.add(metric);
- }
- return metrics.toString();
+ Type metricsType = new TypeToken<List<String>>(){}.getType();
+ String metrics = new Gson().toJson(metricNames, metricsType);
+ return metrics;
}
@GET
@@ -144,11 +144,9 @@ public class MetricsController {
@Produces("application/json")
public String getSourceNames(@Context HttpServletRequest request, @PathParam("metricGroup") String metricGroup) {
Set<String> sourceNames = ChukwaHBaseStore.getSourceNames(metricGroup);
- JSONArray rows = new JSONArray();
- for(String row : sourceNames) {
- rows.add(row);
- }
- return rows.toString();
+ Type rowsType = new TypeToken<List<String>>(){}.getType();
+ String rows = new Gson().toJson(sourceNames, rowsType);
+ return rows;
}
}