You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2020/04/14 11:39:47 UTC
[phoenix] branch master updated: PHOENIX-578 try to standardize on
a JSON library
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new dec131d PHOENIX-578 try to standardize on a JSON library
dec131d is described below
commit dec131d8a4de666c06dabc332383fd7e75c2d3aa
Author: Richard Antal <an...@gmail.com>
AuthorDate: Fri Apr 3 11:10:50 2020 +0200
PHOENIX-578 try to standardize on a JSON library
---
phoenix-core/pom.xml | 12 +++----
.../phoenix/end2end/BackwardCompatibilityIT.java | 21 ++++++-----
.../apache/phoenix/end2end/IndexRebuildTaskIT.java | 2 +-
.../it/resources/compatible_client_versions.json | 3 +-
.../phoenix/coprocessor/TaskRegionObserver.java | 13 +++----
.../coprocessor/tasks/IndexRebuildTask.java | 42 ++++++++++++----------
.../index/automation/PhoenixMRJobSubmitter.java | 26 +++++---------
.../org/apache/phoenix/schema/MetaDataClient.java | 13 ++++---
.../org/apache/phoenix/util/PhoenixMRJobUtil.java | 3 +-
pom.xml | 18 ----------
10 files changed, 64 insertions(+), 89 deletions(-)
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index c119213..66386ad 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -379,6 +379,10 @@
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
@@ -458,14 +462,6 @@
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </dependency>
- <dependency>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
</dependency>
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
index b8580ef..cf28559 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
@@ -41,6 +41,9 @@ import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -65,10 +68,7 @@ import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import com.google.common.collect.Lists;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.stream.JsonReader;
+
/**
* This class is meant for testing all compatible client versions
@@ -139,13 +139,12 @@ public class BackwardCompatibilityIT {
hbaseProfile = m.group();
}
List<String> clientVersions = Lists.newArrayList();
- JsonParser jsonParser = new JsonParser();
- JsonReader jsonReader =
- new JsonReader(new FileReader(COMPATIBLE_CLIENTS_JSON));
- JsonObject jsonObject =
- jsonParser.parse(jsonReader).getAsJsonObject();
- for (JsonElement clientVersion : jsonObject.get(hbaseProfile).getAsJsonArray()) {
- clientVersions.add(clientVersion.getAsString() + "-HBase-" + hbaseProfile);
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
+ JsonNode jsonNode = mapper.readTree(new FileReader(COMPATIBLE_CLIENTS_JSON));
+ JsonNode HBaseProfile = jsonNode.get(hbaseProfile);
+ for (final JsonNode clientVersion : HBaseProfile) {
+ clientVersions.add(clientVersion.textValue() + "-HBase-" + hbaseProfile);
}
return clientVersions;
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexRebuildTaskIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexRebuildTaskIT.java
index 33519b9..f9db41e 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexRebuildTaskIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexRebuildTaskIT.java
@@ -138,7 +138,7 @@ public class IndexRebuildTaskIT extends BaseUniqueNamesOwnClusterIT {
count = getUtility().countRows(indexHTable);
assertEquals(0, count);
- String data = "{IndexName:" + indexName + "}";
+ String data = "{\"IndexName\":\"" + indexName + "\"}";
// Run IndexRebuildTask
TaskRegionObserver.SelfHealingTask task =
diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-core/src/it/resources/compatible_client_versions.json
index 1b436d5..810265c 100644
--- a/phoenix-core/src/it/resources/compatible_client_versions.json
+++ b/phoenix-core/src/it/resources/compatible_client_versions.json
@@ -16,8 +16,7 @@
* limitations under the License.
*/
{
- "_comment": "Lists all phoenix compatible client versions against the current branch version for a given hbase profile \
- If hbase profile is 1.3, phoenix client versions 4.14.3 and 4.15.0 are tested against current branch version",
+ "_comment": "Lists all phoenix compatible client versions against the current branch version for a given hbase profile If hbase profile is 1.3, phoenix client versions 4.14.3 and 4.15.0 are tested against current branch version",
"1.3": ["4.14.3", "4.15.0"],
"1.4": ["4.14.3", "4.15.0"],
"1.5": ["4.15.0"],
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
index 6c846e5..2bab318 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
@@ -32,10 +32,11 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.concurrent.GuardedBy;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
@@ -251,10 +252,10 @@ public class TaskRegionObserver implements RegionObserver, RegionCoprocessor {
if (Strings.isNullOrEmpty(data)) {
data = "{}";
}
- JsonParser jsonParser = new JsonParser();
- JsonObject jsonObject = jsonParser.parse(data).getAsJsonObject();
- jsonObject.addProperty(TASK_DETAILS, taskStatus);
- data = jsonObject.toString();
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode jsonNode = mapper.readTree(data);
+ ((ObjectNode) jsonNode).put(TASK_DETAILS, taskStatus);
+ data = jsonNode.toString();
Timestamp endTs = new Timestamp(EnvironmentEdgeManager.currentTimeMillis());
Task.addTask(connForTask, taskRecord.getTaskType(), taskRecord.getTenantId(), taskRecord.getSchemaName(),
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/tasks/IndexRebuildTask.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/tasks/IndexRebuildTask.java
index 91d7e67..c6dc0b8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/tasks/IndexRebuildTask.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/tasks/IndexRebuildTask.java
@@ -17,9 +17,11 @@
*/
package org.apache.phoenix.coprocessor.tasks;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Strings;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.mapreduce.Cluster;
@@ -65,9 +67,10 @@ public class IndexRebuildTask extends BaseTask {
if (Strings.isNullOrEmpty(taskRecord.getData())) {
data = "{}";
}
- JsonParser jsonParser = new JsonParser();
- JsonObject jsonObject = jsonParser.parse(data).getAsJsonObject();
- String indexName = getIndexName(jsonObject);
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode jsonNode = mapper.readValue(data, JsonNode.class);
+ String indexName = getIndexName(jsonNode);
+
if (Strings.isNullOrEmpty(indexName)) {
String str = "Index name is not found. Index rebuild cannot continue " +
"Data : " + data;
@@ -76,16 +79,16 @@ public class IndexRebuildTask extends BaseTask {
}
boolean shouldDisable = false;
- if (jsonObject.has(DISABLE_BEFORE)) {
- String disableBefore = jsonObject.get(DISABLE_BEFORE).toString();
+ if (jsonNode.has(DISABLE_BEFORE)) {
+ String disableBefore = jsonNode.get(DISABLE_BEFORE).toString();
if (!Strings.isNullOrEmpty(disableBefore)) {
shouldDisable = Boolean.valueOf(disableBefore);
}
}
boolean rebuildAll = false;
- if (jsonObject.has(REBUILD_ALL)) {
- String rebuildAllStr = jsonObject.get(REBUILD_ALL).toString();
+ if (jsonNode.has(REBUILD_ALL)) {
+ String rebuildAllStr = jsonNode.get(REBUILD_ALL).toString();
if (!Strings.isNullOrEmpty(rebuildAllStr)) {
rebuildAll = Boolean.valueOf(rebuildAllStr);
}
@@ -103,9 +106,10 @@ public class IndexRebuildTask extends BaseTask {
Job job = indexToolRes.getValue();
- jsonObject.addProperty(JOB_ID, job.getJobID().toString());
+ ((ObjectNode) jsonNode).put(JOB_ID, job.getJobID().toString());
Task.addTask(conn.unwrap(PhoenixConnection.class ), taskRecord.getTaskType(), taskRecord.getTenantId(), taskRecord.getSchemaName(),
- taskRecord.getTableName(), PTable.TaskStatus.STARTED.toString(), jsonObject.toString(), taskRecord.getPriority(),
+ taskRecord.getTableName(), PTable.TaskStatus.STARTED.toString(),
+ jsonNode.toString(), taskRecord.getPriority(),
taskRecord.getTimeStamp(), null, true);
// It will take some time to finish, so we will check the status in a separate task.
return null;
@@ -129,24 +133,24 @@ public class IndexRebuildTask extends BaseTask {
}
- private String getIndexName(JsonObject jsonObject) {
+ private String getIndexName(JsonNode jsonNode) {
String indexName = null;
// Get index name from data column.
- if (jsonObject.has(INDEX_NAME)) {
- indexName = jsonObject.get(INDEX_NAME).toString().replaceAll("\"", "");
+ if (jsonNode.has(INDEX_NAME)) {
+ indexName = jsonNode.get(INDEX_NAME).toString().replaceAll("\"", "");
}
return indexName;
}
- private String getJobID(String data) {
+ private String getJobID(String data) throws JsonProcessingException {
if (Strings.isNullOrEmpty(data)) {
data = "{}";
}
- JsonParser jsonParser = new JsonParser();
- JsonObject jsonObject = jsonParser.parse(data).getAsJsonObject();
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode jsonNode = mapper.readTree(data);
String jobId = null;
- if (jsonObject.has(JOB_ID)) {
- jobId = jsonObject.get(JOB_ID).toString().replaceAll("\"", "");
+ if (jsonNode.has(JOB_ID)) {
+ jobId = jsonNode.get(JOB_ID).textValue().replaceAll("\"", "");
}
return jobId;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
index 459748c..97c678c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
@@ -39,6 +39,8 @@ import java.util.concurrent.TimeoutException;
import javax.security.auth.login.AppConfigurationEntry;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
@@ -56,15 +58,10 @@ import org.apache.phoenix.util.PhoenixMRJobUtil;
import org.apache.phoenix.util.PhoenixMRJobUtil.MR_SCHEDULER_TYPE;
import org.apache.phoenix.util.UpgradeUtil;
import org.apache.phoenix.util.ZKBasedMasterElectionUtil;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.reflect.TypeToken;
-
public class PhoenixMRJobSubmitter {
@@ -313,25 +310,20 @@ public class PhoenixMRJobSubmitter {
+ "," + YarnApplication.state.RUNNING);
String response = PhoenixMRJobUtil.getJobsInformationFromRM(rmAddress, urlParams);
LOGGER.debug("Already Submitted/Running Apps = " + response);
- JSONObject jobsJson = new JSONObject(response);
- JSONObject appsJson = jobsJson.optJSONObject(YarnApplication.APPS_ELEMENT);
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonNode jsonNode = objectMapper.readTree(response);
+ JsonNode appsJson = jsonNode.get(YarnApplication.APPS_ELEMENT);
Set<String> yarnApplicationSet = new HashSet<String>();
if (appsJson == null) {
return yarnApplicationSet;
}
- JSONArray appJson = appsJson.optJSONArray(YarnApplication.APP_ELEMENT);
+ JsonNode appJson = appsJson.get(YarnApplication.APP_ELEMENT);
if (appJson == null) {
return yarnApplicationSet;
}
- for (int i = 0; i < appJson.length(); i++) {
-
- Gson gson = new GsonBuilder().create();
- YarnApplication yarnApplication =
- gson.fromJson(appJson.getJSONObject(i).toString(),
- new TypeToken<YarnApplication>() {
- }.getType());
- yarnApplicationSet.add(yarnApplication.getName());
+ for (final JsonNode clientVersion : appJson) {
+ yarnApplicationSet.add(clientVersion.get("name").textValue());
}
return yarnApplicationSet;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index b5ba3ef..f6eb1ae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -139,7 +139,7 @@ import java.util.Set;
import java.util.HashSet;
import org.apache.hadoop.conf.Configuration;
-import com.google.gson.JsonObject;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ClusterConnection;
@@ -4514,14 +4514,17 @@ public class MetaDataClient {
tenantId, indexName);
if (tasks == null || tasks.size() == 0) {
Timestamp ts = new Timestamp(EnvironmentEdgeManager.currentTimeMillis());
- JsonObject jsonObject = new JsonObject();
- jsonObject.addProperty(INDEX_NAME, indexName);
- jsonObject.addProperty(REBUILD_ALL, true);
+ Map<String, Object> props = new HashMap<String, Object>() {{
+ put(INDEX_NAME, indexName);
+ put(REBUILD_ALL, true);
+ }};
+ ObjectMapper mapper = new ObjectMapper();
try {
+ String json = mapper.writeValueAsString(props);
Task.addTask(connection, PTable.TaskType.INDEX_REBUILD,
tenantId, schemaName,
dataTableName, PTable.TaskStatus.CREATED.toString(),
- jsonObject.toString(), null, ts, null, true);
+ json, null, ts, null, true);
connection.commit();
} catch (IOException e) {
throw new SQLException("Exception happened while adding a System.Task" + e.toString());
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
index 03d6f3d..dc6a15a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.Activ
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
-import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,7 +89,7 @@ public class PhoenixMRJobUtil {
}
public static String getActiveResourceManagerAddress(Configuration config, String zkQuorum)
- throws IOException, InterruptedException, JSONException, KeeperException,
+ throws IOException, InterruptedException, KeeperException,
InvalidProtocolBufferException, ZooKeeperConnectionException {
// In case of yarn HA is NOT enabled
String resourceManager = PhoenixMRJobUtil.getRMWebAddress(config);
diff --git a/pom.xml b/pom.xml
index 0181548..a3366d6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -117,8 +117,6 @@
<i18n-util.version>1.0.4</i18n-util.version>
<twill.version>0.8.0</twill.version>
<guice.version>4.0</guice.version>
- <gson.version>2.2.4</gson.version>
- <jettison.version>1.1</jettison.version>
<zookeeper.version>3.4.10</zookeeper.version>
<curator.version>4.0.0</curator.version>
<jcodings.version>1.0.18</jcodings.version>
@@ -1192,22 +1190,6 @@
<version>${guava.version}</version>
</dependency>
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>${gson.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- <version>${jettison.version}</version>
- <exclusions>
- <exclusion>
- <groupId>stax</groupId>
- <artifactId>stax-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>${guice.version}</version>