You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by lo...@apache.org on 2015/10/23 12:51:51 UTC
metamodel git commit: METAMODEL-197: Fixed
Repository: metamodel
Updated Branches:
refs/heads/master 60989cc25 -> 9553357cd
METAMODEL-197: Fixed
Fixes #61
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/9553357c
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/9553357c
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/9553357c
Branch: refs/heads/master
Commit: 9553357cd5b1b8e38b774c283d859f6b6f8c62ed
Parents: 60989cc
Author: Dennis Du Krøger <de...@humaninference.com>
Authored: Fri Oct 23 12:45:49 2015 +0200
Committer: Dennis Du Krøger <de...@humaninference.com>
Committed: Fri Oct 23 12:45:49 2015 +0200
----------------------------------------------------------------------
.../apache/metamodel/util/SimpleTableDef.java | 3 +
.../ElasticSearchCreateTableBuilder.java | 3 +-
.../elasticsearch/ElasticSearchDataContext.java | 64 +++++++++++++-------
.../ElasticSearchInsertBuilder.java | 2 +-
.../ElasticSearchDataContextTest.java | 31 ++++++++++
5 files changed, 78 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java b/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java
index 0dd98a9..f9027fb 100644
--- a/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java
+++ b/core/src/main/java/org/apache/metamodel/util/SimpleTableDef.java
@@ -81,6 +81,9 @@ public class SimpleTableDef implements Serializable, HasName {
* the column types of the columns specified.
*/
public SimpleTableDef(String name, String[] columnNames, ColumnType[] columnTypes) {
+ if(name == null){
+ throw new NullPointerException("Table name cannot be null");
+ }
_name = name;
_columnNames = columnNames;
if (columnTypes == null) {
http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java
index e550506..0a1750d 100644
--- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java
+++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchCreateTableBuilder.java
@@ -59,7 +59,7 @@ final class ElasticSearchCreateTableBuilder extends AbstractTableCreationBuilder
final IndicesAdminClient indicesAdmin = dataContext.getElasticSearchClient().admin().indices();
final String indexName = dataContext.getIndexName();
- final List<Object> sourceProperties = new ArrayList<Object>();
+ final List<Object> sourceProperties = new ArrayList<>();
for (Column column : table.getColumns()) {
// each column is defined as a property pair of the form: ("field1",
// "type=string,store=true")
@@ -87,7 +87,6 @@ final class ElasticSearchCreateTableBuilder extends AbstractTableCreationBuilder
final MutableSchema schema = (MutableSchema) getSchema();
schema.addTable(table);
-
return table;
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java
index 6d8d5b4..34bb983 100644
--- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java
+++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContext.java
@@ -93,8 +93,12 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
public static final TimeValue TIMEOUT_SCROLL = TimeValue.timeValueSeconds(60);
private final Client elasticSearchClient;
- private final SimpleTableDef[] tableDefs;
private final String indexName;
+ // Table definitions that are set from the beginning, not supposed to be changed.
+ private final List<SimpleTableDef> staticTableDefinitions;
+
+ // Table definitions that are discovered, these can change
+ private final List<SimpleTableDef> dynamicTableDefinitions = new ArrayList<>();
/**
* Constructs a {@link ElasticSearchDataContext}. This constructor accepts a
@@ -105,11 +109,11 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
* the ElasticSearch client
* @param indexName
* the name of the ElasticSearch index to represent
- * @param tableDefs
+ * @param tableDefinitions
* an array of {@link SimpleTableDef}s, which define the table
* and column model of the ElasticSearch index.
*/
- public ElasticSearchDataContext(Client client, String indexName, SimpleTableDef... tableDefs) {
+ public ElasticSearchDataContext(Client client, String indexName, SimpleTableDef... tableDefinitions) {
if (client == null) {
throw new IllegalArgumentException("ElasticSearch Client cannot be null");
}
@@ -118,13 +122,14 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
}
this.elasticSearchClient = client;
this.indexName = indexName;
- this.tableDefs = tableDefs;
+ this.staticTableDefinitions = Arrays.asList(tableDefinitions);
+ this.dynamicTableDefinitions.addAll(Arrays.asList(detectSchema()));
}
/**
* Constructs a {@link ElasticSearchDataContext} and automatically detects
* the schema structure/view on all indexes (see
- * {@link #detectSchema(Client, String)}).
+ * {@link this.detectSchema(Client, String)}).
*
* @param client
* the ElasticSearch client
@@ -132,7 +137,7 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
* the name of the ElasticSearch index to represent
*/
public ElasticSearchDataContext(Client client, String indexName) {
- this(client, indexName, detectSchema(client, indexName));
+ this(client, indexName, new SimpleTableDef[0]);
}
/**
@@ -141,18 +146,15 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
* based on the metadata provided by the ElasticSearch java client.
*
* @see #detectTable(ClusterState, String, String)
- *
- * @param client
- * the client to inspect
- * @param indexName
* @return a mutable schema instance, useful for further fine tuning by the
* user.
*/
- public static SimpleTableDef[] detectSchema(Client client, String indexName) {
+ private SimpleTableDef[] detectSchema() {
logger.info("Detecting schema for index '{}'", indexName);
final ClusterState cs;
- final ClusterStateRequestBuilder clusterStateRequestBuilder = client.admin().cluster().prepareState();
+ final ClusterStateRequestBuilder clusterStateRequestBuilder =
+ getElasticSearchClient().admin().cluster().prepareState();
// different methods here to set the index name, so we have to use
// reflection :-/
@@ -172,7 +174,7 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
}
cs = clusterStateRequestBuilder.execute().actionGet().getState();
- final List<SimpleTableDef> result = new ArrayList<SimpleTableDef>();
+ final List<SimpleTableDef> result = new ArrayList<>();
final IndexMetaData imd = cs.getMetaData().index(indexName);
if (imd == null) {
@@ -192,7 +194,7 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
}
}
}
- final SimpleTableDef[] tableDefArray = (SimpleTableDef[]) result.toArray(new SimpleTableDef[result.size()]);
+ final SimpleTableDef[] tableDefArray = result.toArray(new SimpleTableDef[result.size()]);
Arrays.sort(tableDefArray, new Comparator<SimpleTableDef>() {
@Override
public int compare(SimpleTableDef o1, SimpleTableDef o2) {
@@ -244,18 +246,36 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
@Override
protected Schema getMainSchema() throws MetaModelException {
final MutableSchema theSchema = new MutableSchema(getMainSchemaName());
- for (final SimpleTableDef tableDef : tableDefs) {
- final MutableTable table = tableDef.toTable().setSchema(theSchema);
- final Column idColumn = table.getColumnByName(FIELD_ID);
- if (idColumn != null && idColumn instanceof MutableColumn) {
- final MutableColumn mutableColumn = (MutableColumn) idColumn;
- mutableColumn.setPrimaryKey(true);
+ for (final SimpleTableDef tableDef : staticTableDefinitions) {
+ addTable(theSchema, tableDef);
+ }
+
+ final SimpleTableDef[] tables = detectSchema();
+ synchronized (this) {
+ dynamicTableDefinitions.clear();
+ dynamicTableDefinitions.addAll(Arrays.asList(tables));
+ for (final SimpleTableDef tableDef : dynamicTableDefinitions) {
+ final List<String> tableNames = Arrays.asList(theSchema.getTableNames());
+
+ if (!tableNames.contains(tableDef.getName())) {
+ addTable(theSchema, tableDef);
+ }
}
- theSchema.addTable(table);
}
+
return theSchema;
}
+ private void addTable(final MutableSchema theSchema, final SimpleTableDef tableDef) {
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ final Column idColumn = table.getColumnByName(FIELD_ID);
+ if (idColumn != null && idColumn instanceof MutableColumn) {
+ final MutableColumn mutableColumn = (MutableColumn) idColumn;
+ mutableColumn.setPrimaryKey(true);
+ }
+ theSchema.addTable(table);
+ }
+
@Override
protected String getMainSchemaName() throws MetaModelException {
return indexName;
@@ -317,7 +337,7 @@ public class ElasticSearchDataContext extends QueryPostprocessDataContext implem
return QueryBuilders.matchAllQuery();
}
- List<QueryBuilder> children = new ArrayList<QueryBuilder>(whereItems.size());
+ List<QueryBuilder> children = new ArrayList<>(whereItems.size());
for (FilterItem item : whereItems) {
final QueryBuilder itemQueryBuilder;
http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java
index 321af1d..f2d8746 100644
--- a/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java
+++ b/elasticsearch/src/main/java/org/apache/metamodel/elasticsearch/ElasticSearchInsertBuilder.java
@@ -47,7 +47,7 @@ final class ElasticSearchInsertBuilder extends AbstractRowInsertionBuilder<Elast
final String documentType = getTable().getName();
final IndexRequestBuilder requestBuilder = new IndexRequestBuilder(client, indexName).setType(documentType);
- final Map<String, Object> valueMap = new HashMap<String, Object>();
+ final Map<String, Object> valueMap = new HashMap<>();
final Column[] columns = getColumns();
final Object[] values = getValues();
for (int i = 0; i < columns.length; i++) {
http://git-wip-us.apache.org/repos/asf/metamodel/blob/9553357c/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java
index efd0c97..5c17587 100644
--- a/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java
+++ b/elasticsearch/src/test/java/org/apache/metamodel/elasticsearch/ElasticSearchDataContextTest.java
@@ -49,10 +49,13 @@ import org.apache.metamodel.schema.Table;
import org.apache.metamodel.update.Update;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequestBuilder;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -214,6 +217,8 @@ public class ElasticSearchDataContextTest {
}
});
+ dataContext.refreshSchemas();
+
try (DataSet ds = dataContext.query().from(table).selectAll().orderBy("bar").execute()) {
assertTrue(ds.next());
assertEquals("hello", ds.getRow().getValue(fooColumn).toString());
@@ -225,6 +230,32 @@ public class ElasticSearchDataContextTest {
}
dataContext.executeUpdate(new DropTable(table));
+
+ dataContext.refreshSchemas();
+
+ assertNull(dataContext.getTableByQualifiedLabel(table.getName()));
+ }
+
+ @Test
+ public void testDetectOutsideChanges() throws Exception {
+ ElasticSearchDataContext elasticSearchDataContext = (ElasticSearchDataContext) dataContext;
+
+ // Create the type in ES
+ final IndicesAdminClient indicesAdmin = elasticSearchDataContext.getElasticSearchClient().admin().indices();
+ final String tableType = "outsideTable";
+
+ Object[] sourceProperties = { "testA", "type=string, store=true", "testB", "type=string, store=true" };
+
+ new PutMappingRequestBuilder(indicesAdmin).setIndices(indexName).setType(tableType).setSource(sourceProperties)
+ .execute().actionGet();
+
+ dataContext.refreshSchemas();
+
+ assertNotNull(dataContext.getDefaultSchema().getTableByName(tableType));
+
+ new DeleteMappingRequestBuilder(indicesAdmin).setIndices(indexName).setType(tableType).execute().actionGet();
+ dataContext.refreshSchemas();
+ assertNull(dataContext.getTableByQualifiedLabel(tableType));
}
@Test