You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2019/02/02 03:14:41 UTC
[incubator-pinot] branch master updated: Add methods
toPrettyJsonString() and toSingleLineJsonString() to Schema (#3783)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e9d4100 Add methods toPrettyJsonString() and toSingleLineJsonString() to Schema (#3783)
e9d4100 is described below
commit e9d4100d0fc331f1ae067973823ff4c3b06a3a5b
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Fri Feb 1 19:14:36 2019 -0800
Add methods toPrettyJsonString() and toSingleLineJsonString() to Schema (#3783)
- For human-read API, use pretty string
- For other purpose, use compact single-line string
Fix the issue of storing pretty string in ZK (consume more space and less readable)
---
.../apache/pinot/common/config/CombinedConfig.java | 8 ++++-
.../java/org/apache/pinot/common/data/Schema.java | 39 ++++++++++++++++------
.../org/apache/pinot/common/utils/SchemaUtils.java | 4 +--
.../org/apache/pinot/common/data/SchemaTest.java | 16 +++++----
.../api/resources/PinotSchemaRestletResource.java | 6 ++--
.../controller/api/resources/PinotTableSchema.java | 2 +-
.../resources/PinotSchemaRestletResourceTest.java | 19 ++++++-----
.../pinot/controller/helix/ControllerTest.java | 7 ++--
.../apache/pinot/hadoop/io/PinotOutputFormat.java | 2 +-
.../pinot/hadoop/job/SegmentCreationJob.java | 2 +-
.../admin/command/AvroSchemaToPinotSchema.java | 2 +-
11 files changed, 66 insertions(+), 41 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/config/CombinedConfig.java b/pinot-common/src/main/java/org/apache/pinot/common/config/CombinedConfig.java
index a668b30..6ffc14c 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/config/CombinedConfig.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/config/CombinedConfig.java
@@ -18,8 +18,10 @@
*/
package org.apache.pinot.common.config;
+import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.common.utils.EqualityUtils;
+import org.apache.pinot.common.utils.JsonUtils;
import static org.apache.pinot.common.utils.EqualityUtils.hashCodeOf;
import static org.apache.pinot.common.utils.EqualityUtils.isEqual;
@@ -43,7 +45,11 @@ public class CombinedConfig {
@Override
public String toString() {
- return "CombinedConfig{" + "_offline=" + _offline + ", _realtime=" + _realtime + ", _schema=" + _schema + '}';
+ try {
+ return JsonUtils.objectToPrettyString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
}
public TableConfig getOfflineTableConfig() {
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/data/Schema.java b/pinot-common/src/main/java/org/apache/pinot/common/data/Schema.java
index b8739f5..f06b14c 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/data/Schema.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/data/Schema.java
@@ -361,43 +361,60 @@ public final class Schema {
return (_timeFieldSpec != null) ? _timeFieldSpec.getOutgoingGranularitySpec().getTimeType() : null;
}
- @JsonIgnore
- @Nonnull
- public String getJSONSchema() {
- ObjectNode jsonSchema = JsonUtils.newObjectNode();
- jsonSchema.put("schemaName", _schemaName);
+ /**
+ * Returns a json representation of the schema.
+ */
+ public ObjectNode toJsonObject() {
+ ObjectNode jsonObject = JsonUtils.newObjectNode();
+ jsonObject.put("schemaName", _schemaName);
if (!_dimensionFieldSpecs.isEmpty()) {
ArrayNode jsonArray = JsonUtils.newArrayNode();
for (DimensionFieldSpec dimensionFieldSpec : _dimensionFieldSpecs) {
jsonArray.add(dimensionFieldSpec.toJsonObject());
}
- jsonSchema.set("dimensionFieldSpecs", jsonArray);
+ jsonObject.set("dimensionFieldSpecs", jsonArray);
}
if (!_metricFieldSpecs.isEmpty()) {
ArrayNode jsonArray = JsonUtils.newArrayNode();
for (MetricFieldSpec metricFieldSpec : _metricFieldSpecs) {
jsonArray.add(metricFieldSpec.toJsonObject());
}
- jsonSchema.set("metricFieldSpecs", jsonArray);
+ jsonObject.set("metricFieldSpecs", jsonArray);
}
if (_timeFieldSpec != null) {
- jsonSchema.set("timeFieldSpec", _timeFieldSpec.toJsonObject());
+ jsonObject.set("timeFieldSpec", _timeFieldSpec.toJsonObject());
}
if (!_dateTimeFieldSpecs.isEmpty()) {
ArrayNode jsonArray = JsonUtils.newArrayNode();
for (DateTimeFieldSpec dateTimeFieldSpec : _dateTimeFieldSpecs) {
jsonArray.add(dateTimeFieldSpec.toJsonObject());
}
- jsonSchema.set("dateTimeFieldSpecs", jsonArray);
+ jsonObject.set("dateTimeFieldSpecs", jsonArray);
}
+ return jsonObject;
+ }
+
+ /**
+ * Returns a pretty json string representation of the schema.
+ */
+ @Nonnull
+ public String toPrettyJsonString() {
try {
- return JsonUtils.objectToPrettyString(jsonSchema);
+ return JsonUtils.objectToPrettyString(toJsonObject());
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
/**
+ * Returns a single-line json string representation of the schema.
+ */
+ @Nonnull
+ public String toSingleLineJsonString() {
+ return toJsonObject().toString();
+ }
+
+ /**
* Validates a pinot schema.
* <p>The following validations are performed:
* <ul>
@@ -622,7 +639,7 @@ public final class Schema {
@Override
public String toString() {
- return getJSONSchema();
+ return toPrettyJsonString();
}
@SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/SchemaUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/SchemaUtils.java
index b0c2016..9b4dd35 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/SchemaUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/SchemaUtils.java
@@ -64,7 +64,7 @@ public class SchemaUtils {
*/
public static ZNRecord toZNRecord(@Nonnull Schema schema) {
ZNRecord record = new ZNRecord(schema.getSchemaName());
- record.setSimpleField("schemaJSON", schema.getJSONSchema());
+ record.setSimpleField("schemaJSON", schema.toSingleLineJsonString());
return record;
}
@@ -118,7 +118,7 @@ public class SchemaUtils {
URL url = new URL("http", host, port, "/schemas");
PostMethod httpPost = new PostMethod(url.toString());
try {
- Part[] parts = {new StringPart(schema.getSchemaName(), schema.toString())};
+ Part[] parts = {new StringPart(schema.getSchemaName(), schema.toSingleLineJsonString())};
MultipartRequestEntity requestEntity = new MultipartRequestEntity(parts, new HttpMethodParams());
httpPost.setRequestEntity(requestEntity);
int responseCode = HTTP_CLIENT.executeMethod(httpPost);
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/data/SchemaTest.java b/pinot-common/src/test/java/org/apache/pinot/common/data/SchemaTest.java
index c3140bb..cd9756a 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/data/SchemaTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/data/SchemaTest.java
@@ -266,7 +266,11 @@ public class SchemaTest {
Assert.assertNotNull(resourceUrl);
Schema schema = Schema.fromFile(new File(resourceUrl.getFile()));
- Schema schemaToCompare = Schema.fromString(schema.getJSONSchema());
+ Schema schemaToCompare = Schema.fromString(schema.toPrettyJsonString());
+ Assert.assertEquals(schemaToCompare, schema);
+ Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode());
+
+ schemaToCompare = Schema.fromString(schema.toSingleLineJsonString());
Assert.assertEquals(schemaToCompare, schema);
Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode());
@@ -275,10 +279,10 @@ public class SchemaTest {
Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode());
// When setting new fields, schema string should be updated
- String JSONSchema = schemaToCompare.getJSONSchema();
+ String jsonSchema = schemaToCompare.toSingleLineJsonString();
schemaToCompare.setSchemaName("newSchema");
- String JSONSchemaToCompare = schemaToCompare.getJSONSchema();
- Assert.assertFalse(JSONSchema.equals(JSONSchemaToCompare));
+ String jsonSchemaToCompare = schemaToCompare.toSingleLineJsonString();
+ Assert.assertNotEquals(jsonSchemaToCompare, jsonSchema);
}
@Test
@@ -292,7 +296,7 @@ public class SchemaTest {
TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date");
Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema")
.addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec).build();
- String jsonSchema = schema.getJSONSchema();
+ String jsonSchema = schema.toSingleLineJsonString();
Schema schemaFromJson = Schema.fromString(jsonSchema);
Assert.assertEquals(schemaFromJson, schema);
Assert.assertEquals(schemaFromJson.hashCode(), schema.hashCode());
@@ -311,7 +315,7 @@ public class SchemaTest {
expectedSchema.addField(new MetricFieldSpec("nonEmptyDefault", FieldSpec.DataType.BYTES, expectedNonEmptyDefault));
// Ensure that schema can be serialized and de-serialized (ie byte[] converted to String and back).
- String jsonSchema = expectedSchema.getJSONSchema();
+ String jsonSchema = expectedSchema.toSingleLineJsonString();
Schema actualSchema = Schema.fromString(jsonSchema);
Assert.assertEquals(actualSchema.getFieldSpecFor("noDefault").getDefaultNullValue(), expectedEmptyDefault);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
index ec743bb..82a4841 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java
@@ -95,9 +95,7 @@ public class PinotSchemaRestletResource {
if (schema == null) {
throw new ControllerApplicationException(LOGGER, "Schema not found", Response.Status.NOT_FOUND);
}
- // We need to return schema.getJSONSchema(). Returning schema ends up with many extra fields, "jsonSchema" being one of them,
- // Others like fieldSpecMap, etc., serialzing the entire Schema object.
- return schema.getJSONSchema();
+ return schema.toPrettyJsonString();
}
@DELETE
@@ -144,7 +142,7 @@ public class PinotSchemaRestletResource {
throw new ControllerApplicationException(LOGGER, "Invalid schema. Check controller logs",
Response.Status.BAD_REQUEST);
}
- return schema.getJSONSchema();
+ return schema.toPrettyJsonString();
}
/**
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableSchema.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableSchema.java
index 3375ba1..6287644 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableSchema.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableSchema.java
@@ -53,7 +53,7 @@ public class PinotTableSchema {
@ApiParam(value = "Table name (without type)", required = true) @PathParam("tableName") String tableName) {
Schema schema = pinotHelixResourceManager.getTableSchema(tableName);
if (schema != null) {
- return schema.getJSONSchema();
+ return schema.toPrettyJsonString();
}
throw new ControllerApplicationException(LOGGER, "Schema not found for table: " + tableName,
Response.Status.NOT_FOUND);
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResourceTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResourceTest.java
index 16369ad..68dc4ee 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResourceTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResourceTest.java
@@ -43,7 +43,7 @@ public class PinotSchemaRestletResourceTest extends ControllerTest {
public void testBadContentType() {
Schema schema = createDummySchema("testSchema");
try {
- sendPostRequest(_controllerRequestURLBuilder.forSchemaCreate(), schema.getJSONSchema());
+ sendPostRequest(_controllerRequestURLBuilder.forSchemaCreate(), schema.toSingleLineJsonString());
} catch (IOException e) {
// TODO The Jersey API returns 400, so we need to check return code here not a string.
// Assert.assertTrue(e.getMessage().startsWith("Server returned HTTP response code: 415"), e.getMessage());
@@ -59,40 +59,41 @@ public class PinotSchemaRestletResourceTest extends ControllerTest {
String schemaName = "testSchema";
Schema schema = createDummySchema(schemaName);
String url = _controllerRequestURLBuilder.forSchemaCreate();
- PostMethod postMethod = sendMultipartPostRequest(url, schema.toString());
+ PostMethod postMethod = sendMultipartPostRequest(url, schema.toSingleLineJsonString());
Assert.assertEquals(postMethod.getStatusCode(), 200);
schema.addField(new DimensionFieldSpec("NewColumn", FieldSpec.DataType.STRING, true));
- postMethod = sendMultipartPostRequest(url, schema.toString());
+ postMethod = sendMultipartPostRequest(url, schema.toSingleLineJsonString());
Assert.assertEquals(postMethod.getStatusCode(), 200);
String schemaStr = sendGetRequest(_controllerRequestURLBuilder.forSchemaGet(schemaName));
Schema readSchema = Schema.fromString(schemaStr);
- Schema inputSchema = Schema.fromString(schema.toString());
+ Schema inputSchema = Schema.fromString(schema.toSingleLineJsonString());
Assert.assertEquals(readSchema, inputSchema);
Assert.assertTrue(readSchema.getFieldSpecMap().containsKey("NewColumn"));
final String yetAnotherColumn = "YetAnotherColumn";
Assert.assertFalse(readSchema.getFieldSpecMap().containsKey(yetAnotherColumn));
schema.addField(new DimensionFieldSpec(yetAnotherColumn, FieldSpec.DataType.STRING, true));
- PutMethod putMethod =
- sendMultipartPutRequest(_controllerRequestURLBuilder.forSchemaUpdate(schemaName), schema.toString());
+ PutMethod putMethod = sendMultipartPutRequest(_controllerRequestURLBuilder.forSchemaUpdate(schemaName),
+ schema.toSingleLineJsonString());
Assert.assertEquals(putMethod.getStatusCode(), 200);
// verify some more...
schemaStr = sendGetRequest(_controllerRequestURLBuilder.forSchemaGet(schemaName));
readSchema = Schema.fromString(schemaStr);
- inputSchema = Schema.fromString(schema.toString());
+ inputSchema = Schema.fromString(schema.toSingleLineJsonString());
Assert.assertEquals(readSchema, inputSchema);
Assert.assertTrue(readSchema.getFieldSpecMap().containsKey(yetAnotherColumn));
// error cases
putMethod = sendMultipartPutRequest(_controllerRequestURLBuilder.forSchemaUpdate(schemaName),
- schema.toString().substring(1));
+ schema.toSingleLineJsonString().substring(1));
// invalid json
Assert.assertEquals(putMethod.getStatusCode(), 400);
schema.setSchemaName("differentSchemaName");
- putMethod = sendMultipartPutRequest(_controllerRequestURLBuilder.forSchemaUpdate(schemaName), schema.toString());
+ putMethod = sendMultipartPutRequest(_controllerRequestURLBuilder.forSchemaUpdate(schemaName),
+ schema.toSingleLineJsonString());
Assert.assertEquals(putMethod.getStatusCode(), 400);
}
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
index 41fa3f7..b3d6d31 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java
@@ -167,17 +167,16 @@ public abstract class ControllerTest {
protected void addDummySchema(String tableName)
throws IOException {
- addSchema(createDummySchema(tableName).getJSONSchema());
+ addSchema(createDummySchema(tableName));
}
/**
* Add a schema to the controller.
- * @param schemaJson the json string representing the schema
*/
- protected void addSchema(String schemaJson)
+ protected void addSchema(Schema schema)
throws IOException {
String url = _controllerRequestURLBuilder.forSchemaCreate();
- PostMethod postMethod = sendMultipartPostRequest(url, schemaJson);
+ PostMethod postMethod = sendMultipartPostRequest(url, schema.toSingleLineJsonString());
Assert.assertEquals(postMethod.getStatusCode(), 200);
}
diff --git a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/PinotOutputFormat.java b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/PinotOutputFormat.java
index bca0e73..ecbac96 100644
--- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/PinotOutputFormat.java
+++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/PinotOutputFormat.java
@@ -116,7 +116,7 @@ public class PinotOutputFormat<K, V> extends FileOutputFormat<K, V> {
}
public static void setSchema(Job job, Schema schema) {
- job.getConfiguration().set(PinotOutputFormat.SCHEMA, schema.getJSONSchema());
+ job.getConfiguration().set(PinotOutputFormat.SCHEMA, schema.toSingleLineJsonString());
}
public static String getSchema(JobContext context) {
diff --git a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
index fe9f8fd..7570600 100644
--- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
+++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
@@ -210,7 +210,7 @@ public class SegmentCreationJob extends Configured {
job.getConfiguration().setInt(JobContext.NUM_MAPS, inputDataFiles.size());
if (_dataSchema != null) {
- job.getConfiguration().set(JobConfigConstants.SCHEMA, _dataSchema.toString());
+ job.getConfiguration().set(JobConfigConstants.SCHEMA, _dataSchema.toSingleLineJsonString());
}
setOutputPath(job.getConfiguration());
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AvroSchemaToPinotSchema.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AvroSchemaToPinotSchema.java
index f5b9500..2aaa5c6 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AvroSchemaToPinotSchema.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AvroSchemaToPinotSchema.java
@@ -97,7 +97,7 @@ public class AvroSchemaToPinotSchema extends AbstractBaseAdminCommand implements
LOGGER.info("Store Pinot schema to file: {}", outputFile.getAbsolutePath());
try (FileWriter writer = new FileWriter(outputFile)) {
- writer.write(schema.toString());
+ writer.write(schema.toPrettyJsonString());
}
return true;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org