You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ka...@apache.org on 2015/08/13 17:21:52 UTC
[1/2] metamodel git commit: METAMODEL-164: Fixed and refactored ES
mapping parser to use real map.
Repository: metamodel
Updated Branches:
refs/heads/master 96f577774 -> b6fb6e04e
METAMODEL-164: Fixed and refactored ES mapping parser to use real map.
Previously the parser was (by accident I think) using the toString()
representation of that same map which made it very vulnerable towards
parsing issues.
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/6f4958f1
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/6f4958f1
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/6f4958f1
Branch: refs/heads/master
Commit: 6f4958f1e916c8cf4560da8cb04cfb53ff578764
Parents: 96f5777
Author: Kasper Sørensen <i....@gmail.com>
Authored: Wed Aug 12 22:56:52 2015 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Wed Aug 12 22:56:52 2015 +0200
----------------------------------------------------------------------
.../elasticsearch/ElasticSearchDataContext.java | 24 ++++----
.../ElasticSearchMetaDataParser.java | 63 +++++++++-----------
.../ElasticSearchMetaDataParserTest.java | 30 ++++++----
3 files changed, 57 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/6f4958f1/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java
index ce028f6..ba221d1 100644
--- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java
+++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java
@@ -22,7 +22,6 @@ import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -228,20 +227,17 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
throw new IllegalArgumentException("No such document type in index '" + indexName + "': " + documentType);
}
final Map<String, Object> mp = mappingMetaData.getSourceAsMap();
- final Iterator<Map.Entry<String, Object>> it = mp.entrySet().iterator();
- SimpleTableDef std = null;
- while (it.hasNext()) {
- final Map.Entry<String, Object> pair = it.next();
- if (pair.getKey().equals("properties")) {
- final ElasticSearchMetaData metaData = ElasticSearchMetaDataParser.parse(pair.getValue());
- std = new SimpleTableDef(documentType, metaData.getColumnNames(), metaData.getColumnTypes());
- }
- }
- if (std == null) {
- throw new IllegalArgumentException("No properties defined for document type '" + documentType
- + "' in index: " + indexName);
+ final Object metadataProperties = mp.get("properties");
+ if (metadataProperties != null && metadataProperties instanceof Map) {
+ @SuppressWarnings("unchecked")
+ final Map<String, ?> metadataPropertiesMap = (Map<String, ?>) metadataProperties;
+ final ElasticSearchMetaData metaData = ElasticSearchMetaDataParser.parse(metadataPropertiesMap);
+ final SimpleTableDef std = new SimpleTableDef(documentType, metaData.getColumnNames(),
+ metaData.getColumnTypes());
+ return std;
}
- return std;
+ throw new IllegalArgumentException("No mapping properties defined for document type '" + documentType
+ + "' in index: " + indexName);
}
@Override
http://git-wip-us.apache.org/repos/asf/metamodel/blob/6f4958f1/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java
index 82953d1..41cf184 100644
--- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java
+++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParser.java
@@ -18,6 +18,9 @@
*/
package org.apache.metamodel.elasticsearch;
+import java.util.Map;
+import java.util.Map.Entry;
+
import org.apache.metamodel.schema.ColumnType;
/**
@@ -31,51 +34,42 @@ public class ElasticSearchMetaDataParser {
* object. This method makes much easier to create the ElasticSearch schema.
*
* @param metaDataInfo
- * ElasticSearch metadata info in a json-like format
+ * ElasticSearch mapping metadata in Map format
* @return An ElasticSearchMetaData object
*/
- public static ElasticSearchMetaData parse(Object metaDataInfo) {
- final String plainMetaDataInfo = removeFirstAndLastCharacter(metaDataInfo.toString());
- final String metaDataWithoutDateFormats = removeDateFormats(plainMetaDataInfo);
- final String[] metaDataFields = metaDataWithoutDateFormats.split("},");
-
- final String[] fieldNames = new String[metaDataFields.length + 1];
- final ColumnType[] columnTypes = new ColumnType[metaDataFields.length + 1];
+ public static ElasticSearchMetaData parse(Map<String, ?> metaDataInfo) {
+ final String[] fieldNames = new String[metaDataInfo.size() + 1];
+ final ColumnType[] columnTypes = new ColumnType[metaDataInfo.size() + 1];
// add the document ID field (fixed)
fieldNames[0] = ElasticSearchDataContext.FIELD_ID;
columnTypes[0] = ColumnType.STRING;
int i = 1;
- for (String metaDataField : metaDataFields) {
- // message={type=long}
- fieldNames[i] = getNameFromMetaDataField(metaDataField);
- columnTypes[i] = getColumnTypeFromMetaDataField(metaDataField);
+ for (Entry<String, ?> metaDataField : metaDataInfo.entrySet()) {
+ @SuppressWarnings("unchecked")
+ final Map<String, ?> fieldMetadata = (Map<String, ?>) metaDataField.getValue();
+
+ fieldNames[i] = metaDataField.getKey();
+ columnTypes[i] = getColumnTypeFromMetadataField(fieldMetadata);
i++;
}
return new ElasticSearchMetaData(fieldNames, columnTypes);
}
- private static String removeFirstAndLastCharacter(String metaDataInfo) {
- return metaDataInfo.substring(1, metaDataInfo.length() - 1);
- }
-
- private static String removeDateFormats(String metaDataInfo) {
- return metaDataInfo.replaceAll("type=date.*?}", "type=date}");
- }
+ private static ColumnType getColumnTypeFromMetadataField(Map<String, ?> fieldMetadata) {
+ final ColumnType columnType;
+ final String metaDataFieldType = getMetaDataFieldTypeFromMetaDataField(fieldMetadata);
- private static String getNameFromMetaDataField(String metaDataField) {
- return metaDataField.substring(0, metaDataField.indexOf("=")).trim();
- }
+ if (metaDataFieldType == null) {
+ return ColumnType.STRING;
+ }
- private static ColumnType getColumnTypeFromMetaDataField(String metaDataField) {
- final ColumnType columnType;
- final String metaDataFieldType = getMetaDataFieldTypeFromMetaDataField(metaDataField);
- if (metaDataFieldType.equals("long")) {
- columnType = ColumnType.BIGINT;
- } else if (metaDataFieldType.equals("date")) {
+ if (metaDataFieldType.startsWith("date")) {
columnType = ColumnType.DATE;
+ } else if (metaDataFieldType.equals("long")) {
+ columnType = ColumnType.BIGINT;
} else if (metaDataFieldType.equals("string")) {
columnType = ColumnType.STRING;
} else if (metaDataFieldType.equals("float")) {
@@ -90,15 +84,12 @@ public class ElasticSearchMetaDataParser {
return columnType;
}
- private static String getMetaDataFieldTypeFromMetaDataField(String metaDataField) {
- String type = metaDataField.substring(metaDataField.indexOf("type=") + 5);
- if (type.indexOf(",") > 0) {
- type = type.substring(0, type.indexOf(","));
- }
- if (type.indexOf("}") > 0) {
- type = type.substring(0, type.indexOf("}"));
+ private static String getMetaDataFieldTypeFromMetaDataField(Map<String, ?> metaDataField) {
+ final Object type = metaDataField.get("type");
+ if (type == null) {
+ return null;
}
- return type;
+ return type.toString();
}
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/6f4958f1/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java
index 48a1f06..4d8bef3 100644
--- a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java
+++ b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchMetaDataParserTest.java
@@ -18,24 +18,31 @@
*/
package org.apache.metamodel.elasticsearch;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
import junit.framework.TestCase;
+
import org.apache.metamodel.schema.ColumnType;
+import org.elasticsearch.common.collect.MapBuilder;
public class ElasticSearchMetaDataParserTest extends TestCase {
public void testParseMetadataInfo() throws Exception {
- String metaDataInfo = "{message={type=long}, " +
- "postDate={type=date, format=dateOptionalTime}, " +
- "anotherDate={type=date, format=dateOptionalTime}, " +
- "user={type=string}, " +
- "critical={type=boolean}, " +
- "income={type=double}}";
-
- ElasticSearchMetaData metaData = ElasticSearchMetaDataParser.parse(metaDataInfo);
+ Map<String, Object> metadata = new LinkedHashMap<String, Object>();
+ metadata.put("message", MapBuilder.newMapBuilder().put("type", "long").immutableMap());
+ metadata.put("postDate", MapBuilder.newMapBuilder().put("type", "date").put("format", "dateOptionalTime").immutableMap());
+ metadata.put("anotherDate", MapBuilder.newMapBuilder().put("type", "date").put("format", "dateOptionalTime").immutableMap());
+ metadata.put("user", MapBuilder.newMapBuilder().put("type", "string").immutableMap());
+ metadata.put("critical", MapBuilder.newMapBuilder().put("type", "boolean").immutableMap());
+ metadata.put("income", MapBuilder.newMapBuilder().put("type", "double").immutableMap());
+ metadata.put("untypedthingie", MapBuilder.newMapBuilder().put("foo", "bar").immutableMap());
+
+ ElasticSearchMetaData metaData = ElasticSearchMetaDataParser.parse(metadata);
String[] columnNames = metaData.getColumnNames();
ColumnType[] columnTypes = metaData.getColumnTypes();
- assertTrue(columnNames.length == 7);
+ assertTrue(columnNames.length == 8);
assertEquals(columnNames[0], "_id");
assertEquals(columnNames[1], "message");
assertEquals(columnNames[2], "postDate");
@@ -43,7 +50,9 @@ public class ElasticSearchMetaDataParserTest extends TestCase {
assertEquals(columnNames[4], "user");
assertEquals(columnNames[5], "critical");
assertEquals(columnNames[6], "income");
- assertTrue(columnTypes.length == 7);
+ assertEquals(columnNames[7], "untypedthingie");
+
+ assertTrue(columnTypes.length == 8);
assertEquals(columnTypes[0], ColumnType.STRING);
assertEquals(columnTypes[1], ColumnType.BIGINT);
assertEquals(columnTypes[2], ColumnType.DATE);
@@ -51,5 +60,6 @@ public class ElasticSearchMetaDataParserTest extends TestCase {
assertEquals(columnTypes[4], ColumnType.STRING);
assertEquals(columnTypes[5], ColumnType.BOOLEAN);
assertEquals(columnTypes[6], ColumnType.DOUBLE);
+ assertEquals(columnTypes[7], ColumnType.STRING);
}
}
[2/2] metamodel git commit: Updated CHANGES.md
Posted by ka...@apache.org.
Updated CHANGES.md
Fixes #38
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/b6fb6e04
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/b6fb6e04
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/b6fb6e04
Branch: refs/heads/master
Commit: b6fb6e04e8090962bc96fa631ad5880e73a78ae1
Parents: 6f4958f
Author: Kasper Sørensen <i....@gmail.com>
Authored: Thu Aug 13 17:21:11 2015 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Thu Aug 13 17:21:11 2015 +0200
----------------------------------------------------------------------
CHANGES.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/b6fb6e04/CHANGES.md
----------------------------------------------------------------------
diff --git a/CHANGES.md b/CHANGES.md
index 9271960..f0a11a0 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -5,6 +5,7 @@
* [METAMODEL-162] - Made HdfsResource Serializable and added property getters
* [METAMODEL-163] - Made FileResource and HdfsResource work with folders containing file chunks instead of only single files
* [METAMODEL-104] - Added 'hadoop' and 'hbase' modules to dependencies of 'MetaModel-full'.
+ * [METAMODEL-164] - Fixed a bug in data type parsing of ElasticSearch mapping document.
### Apache MetaModel 4.3.5