You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ka...@apache.org on 2014/07/11 21:37:06 UTC
[2/3] git commit: METAMODEL-65: Removed and replaced the
detectSchema(...) and detectTable(...) methods on CouchDbDataContext with an
alternative constructor
METAMODEL-65: Removed and replaced the detectSchema(...) and
detectTable(...) methods on CouchDbDataContext with an alternative
constructor
Project: http://git-wip-us.apache.org/repos/asf/incubator-metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metamodel/commit/b4c6a77b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metamodel/tree/b4c6a77b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metamodel/diff/b4c6a77b
Branch: refs/heads/master
Commit: b4c6a77bf1536e15987dbbfb946de5ba90828056
Parents: 12bdbb6
Author: Kasper Sørensen <i....@gmail.com>
Authored: Wed Jul 9 23:11:50 2014 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Wed Jul 9 23:11:50 2014 +0200
----------------------------------------------------------------------
.../metamodel/convert/DocumentConverter.java | 44 ++++++
.../org/apache/metamodel/data/Document.java | 59 +++++++
.../apache/metamodel/data/DocumentSource.java | 39 +++++
.../metamodel/data/DocumentSourceDataSet.java | 69 +++++++++
.../metamodel/data/MaxRowsDocumentSource.java | 32 ++++
.../builder/ColumnNameAsKeysRowConverter.java | 11 +-
.../schema/builder/DocumentConverter.java | 41 -----
.../schema/builder/DocumentSource.java | 39 -----
.../schema/builder/DocumentSourceProvider.java | 45 ++++++
.../builder/InferentialSchemaBuilder.java | 56 +++++--
.../schema/builder/InferentialTableBuilder.java | 9 +-
.../schema/builder/LazyDocumentSource.java | 6 +-
.../MultiTableInferentialSchemaBuilder.java | 34 +++-
.../metamodel/schema/builder/SchemaBuilder.java | 18 ++-
.../builder/SimpleTableDefSchemaBuilder.java | 8 +-
.../builder/SingleMapColumnSchemaBuilder.java | 13 +-
.../SingleTableInferentialSchemaBuilder.java | 10 +-
.../metamodel/schema/builder/TableBuilder.java | 1 +
.../metamodel/couchdb/CouchDbDataContext.java | 154 ++++---------------
.../couchdb/CouchDbDatabaseDocumentSource.java | 85 ++++++++++
.../couchdb/CouchDbDocumentConverter.java | 18 +++
.../CouchDbInferentialSchemaBuilder.java | 72 +++++++++
.../couchdb/CouchDbSamplingDocumentSource.java | 81 ++++++++++
.../CouchDbSimpleTableDefSchemaBuilder.java | 55 +++++++
.../couchdb/CouchDbTableCreationBuilder.java | 13 +-
.../apache/metamodel/couchdb/CouchDbUtils.java | 17 ++
.../couchdb/CouchDbDataContextTest.java | 56 +++++--
.../apache/metamodel/json/JsonDataContext.java | 49 +++---
.../org/apache/metamodel/json/JsonDataSet.java | 71 ---------
.../metamodel/json/JsonDocumentSource.java | 14 +-
30 files changed, 848 insertions(+), 371 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/convert/DocumentConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/convert/DocumentConverter.java b/core/src/main/java/org/apache/metamodel/convert/DocumentConverter.java
new file mode 100644
index 0000000..f3669c4
--- /dev/null
+++ b/core/src/main/java/org/apache/metamodel/convert/DocumentConverter.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.convert;
+
+import java.util.Map;
+
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.schema.builder.SchemaBuilder;
+
+/**
+ * Object responsible for converting a document ( {@link Map}) into a
+ * {@link Row} for a {@link DataSet} that is based on a {@link SchemaBuilder}.
+ */
+public interface DocumentConverter {
+
+ /**
+ * Converts a {@link Document} into a row with the given
+ * {@link DataSetHeader}.
+ *
+ * @param document
+ * @param header
+ * @return
+ */
+ public Row convert(Document document, DataSetHeader header);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/data/Document.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/data/Document.java b/core/src/main/java/org/apache/metamodel/data/Document.java
new file mode 100644
index 0000000..60cc080
--- /dev/null
+++ b/core/src/main/java/org/apache/metamodel/data/Document.java
@@ -0,0 +1,59 @@
+package org.apache.metamodel.data;
+
+import java.util.Map;
+
+import org.apache.metamodel.convert.DocumentConverter;
+
+/**
+ * Represents a document, ie. an object to be turned into a {@link Row} using a
+ * {@link DocumentConverter} and to be sourced by a {@link DocumentSource}.
+ *
+ * A document does not require any schema. A document will hold key/value pairs
+ * where keys are always strings, but values may be arbitrary values.
+ */
+public class Document {
+
+ private final Map<String, ?> _values;
+ private final Object _sourceObject;
+ private final String _sourceCollectionName;
+
+ public Document(Map<String, ?> values, Object sourceObject) {
+ this(null, values, sourceObject);
+ }
+
+ public Document(String sourceCollectionName, Map<String, ?> values, Object sourceObject) {
+ _sourceCollectionName = sourceCollectionName;
+ _values = values;
+ _sourceObject = sourceObject;
+ }
+
+ /**
+ * Gets the values of the document.
+ *
+ * @return
+ */
+ public Map<String, ?> getValues() {
+ return _values;
+ }
+
+ /**
+ * Gets the source representation of the document, if any.
+ *
+ * @return
+ */
+ public Object getSourceObject() {
+ return _sourceObject;
+ }
+
+ /**
+ * Gets the collection/table name as defined in the source, or a hint about
+ * a table name of this document. This method may return null if the
+ * {@link DocumentSource} does not have any knowledge about the originating
+ * collection name, or if there is no logical way to determine such a name.
+ *
+ * @return
+ */
+ public String getSourceCollectionName() {
+ return _sourceCollectionName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/data/DocumentSource.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/data/DocumentSource.java b/core/src/main/java/org/apache/metamodel/data/DocumentSource.java
new file mode 100644
index 0000000..13bc952
--- /dev/null
+++ b/core/src/main/java/org/apache/metamodel/data/DocumentSource.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.data;
+
+import java.io.Closeable;
+
+
+/**
+ * A source of documents to be used for building/detecting a schema or table.
+ */
+public interface DocumentSource extends Closeable {
+
+ /**
+ * Gets the next sample from the source, or returns null if there are no
+ * more documents available.
+ *
+ * @return
+ */
+ public Document next();
+
+ @Override
+ public void close();
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/data/DocumentSourceDataSet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/data/DocumentSourceDataSet.java b/core/src/main/java/org/apache/metamodel/data/DocumentSourceDataSet.java
new file mode 100644
index 0000000..7ee7df1
--- /dev/null
+++ b/core/src/main/java/org/apache/metamodel/data/DocumentSourceDataSet.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.data;
+
+import org.apache.metamodel.convert.DocumentConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A {@link DataSet} that uses a {@link DocumentSource} as it's source.
+ */
+public class DocumentSourceDataSet extends AbstractDataSet {
+
+ private static final Logger logger = LoggerFactory.getLogger(DocumentSourceDataSet.class);
+
+ private final DocumentSource _documentSource;
+ private final DocumentConverter _converter;
+ private volatile Document _document;
+
+ public DocumentSourceDataSet(DataSetHeader header, DocumentSource documentSource, DocumentConverter converter) {
+ super(header);
+ _documentSource = documentSource;
+ _converter = converter;
+ }
+
+ @Override
+ public boolean next() {
+ _document = _documentSource.next();
+ if (_document == null) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public Row getRow() {
+ if (_document == null) {
+ return null;
+ }
+ final DataSetHeader header = getHeader();
+ return _converter.convert(_document, header);
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ try {
+ _documentSource.close();
+ } catch (Exception e) {
+ logger.warn("Failed to close DocumentSource: {}", _document, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/data/MaxRowsDocumentSource.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/data/MaxRowsDocumentSource.java b/core/src/main/java/org/apache/metamodel/data/MaxRowsDocumentSource.java
new file mode 100644
index 0000000..b0cfdbc
--- /dev/null
+++ b/core/src/main/java/org/apache/metamodel/data/MaxRowsDocumentSource.java
@@ -0,0 +1,32 @@
+package org.apache.metamodel.data;
+
+/**
+ * A {@link DocumentSource} that has a max rows condition on it, that will make
+ * it stop serving documents after a certain limit.
+ */
+public class MaxRowsDocumentSource implements DocumentSource {
+
+ private final DocumentSource _delegate;
+ private volatile int _rowsLeft;
+
+ public MaxRowsDocumentSource(DocumentSource delegate, int maxRows) {
+ _delegate = delegate;
+ _rowsLeft = maxRows;
+ }
+
+ @Override
+ public Document next() {
+ if (_rowsLeft > 0) {
+ Document next = _delegate.next();
+ _rowsLeft--;
+ return next;
+ }
+ return null;
+ }
+
+ @Override
+ public void close() {
+ _delegate.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/ColumnNameAsKeysRowConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/ColumnNameAsKeysRowConverter.java b/core/src/main/java/org/apache/metamodel/schema/builder/ColumnNameAsKeysRowConverter.java
index 7337da0..f50db02 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/ColumnNameAsKeysRowConverter.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/ColumnNameAsKeysRowConverter.java
@@ -20,8 +20,10 @@ package org.apache.metamodel.schema.builder;
import java.util.Map;
+import org.apache.metamodel.convert.DocumentConverter;
import org.apache.metamodel.data.DataSetHeader;
import org.apache.metamodel.data.DefaultRow;
+import org.apache.metamodel.data.Document;
import org.apache.metamodel.data.Row;
/**
@@ -31,13 +33,18 @@ import org.apache.metamodel.data.Row;
public class ColumnNameAsKeysRowConverter implements DocumentConverter {
@Override
- public Row convert(Map<String, ?> document, DataSetHeader header) {
+ public Row convert(Document document, DataSetHeader header) {
final Object[] values = new Object[header.size()];
for (int i = 0; i < values.length; i++) {
final String columnName = header.getSelectItem(i).getColumn().getName();
- values[i] = document.get(columnName);
+ values[i] = get(document, columnName);
}
return new DefaultRow(header, values);
}
+ protected Object get(Document document, String columnName) {
+ final Map<String, ?> map = document.getValues();
+ return map.get(columnName);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/DocumentConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/DocumentConverter.java b/core/src/main/java/org/apache/metamodel/schema/builder/DocumentConverter.java
deleted file mode 100644
index accefe6..0000000
--- a/core/src/main/java/org/apache/metamodel/schema/builder/DocumentConverter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.metamodel.schema.builder;
-
-import java.util.Map;
-
-import org.apache.metamodel.data.DataSet;
-import org.apache.metamodel.data.DataSetHeader;
-import org.apache.metamodel.data.Row;
-
-/**
- * Object responsible for converting a document ( {@link Map}) into a
- * {@link Row} for a {@link DataSet} that is based on a {@link SchemaBuilder}.
- */
-public interface DocumentConverter {
-
- /**
- * Converts a document into a row with the given {@link DataSetHeader}.
- *
- * @param document
- * @param header
- * @return
- */
- public Row convert(Map<String, ?> document, DataSetHeader header);
-}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/DocumentSource.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/DocumentSource.java b/core/src/main/java/org/apache/metamodel/schema/builder/DocumentSource.java
deleted file mode 100644
index da230d8..0000000
--- a/core/src/main/java/org/apache/metamodel/schema/builder/DocumentSource.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.metamodel.schema.builder;
-
-import java.io.Closeable;
-import java.util.Map;
-
-/**
- * A source of documents to be used for building/detecting a schema or table.
- */
-public interface DocumentSource extends Closeable {
-
- /**
- * Gets the next sample from the source, or returns null if there are no
- * more documents available.
- *
- * @return
- */
- public Map<String, ?> next();
-
- @Override
- public void close();
-}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/DocumentSourceProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/DocumentSourceProvider.java b/core/src/main/java/org/apache/metamodel/schema/builder/DocumentSourceProvider.java
new file mode 100644
index 0000000..355865d
--- /dev/null
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/DocumentSourceProvider.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.schema.builder;
+
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.DocumentSource;
+
+/**
+ * A provider of {@link DocumentSource}s for building schemas
+ */
+public interface DocumentSourceProvider {
+
+ /**
+ * Gets a {@link DocumentSource} containing documents of mixed origin and
+ * type.
+ *
+ * @return
+ */
+ public DocumentSource getMixedDocumentSourceForSampling();
+
+ /**
+ * Gets a {@link DocumentSource} for a particular source collection. See
+ * {@link Document#getSourceCollectionName()}.
+ *
+ * @param sourceCollectionName
+ * @return
+ */
+ public DocumentSource getDocumentSourceForTable(String sourceCollectionName);
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/InferentialSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/InferentialSchemaBuilder.java b/core/src/main/java/org/apache/metamodel/schema/builder/InferentialSchemaBuilder.java
index 90094ec..84dedf1 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/InferentialSchemaBuilder.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/InferentialSchemaBuilder.java
@@ -18,12 +18,13 @@
*/
package org.apache.metamodel.schema.builder;
-import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.DocumentSource;
import org.apache.metamodel.schema.MutableSchema;
import org.apache.metamodel.schema.MutableTable;
@@ -38,28 +39,53 @@ public abstract class InferentialSchemaBuilder implements SchemaBuilder {
}
@Override
- public void offerSource(DocumentSource documentSource) {
- while (true) {
- Map<String, ?> map = documentSource.next();
- if (map == null) {
- return;
+ public void offerSources(DocumentSourceProvider documentSourceProvider) {
+ final DocumentSource documentSource = documentSourceProvider.getMixedDocumentSourceForSampling();
+ try {
+ while (true) {
+ final Document document = documentSource.next();
+ if (document == null) {
+ break;
+ }
+ final String tableName = determineTable(document);
+ addObservation(tableName, document);
}
- String tableName = determineTable(map);
- addObservation(tableName, map);
+ } finally {
+ documentSource.close();
}
}
+ protected void offerDocumentSource(DocumentSource documentSource) {
+ try {
+ while (true) {
+ final Document document = documentSource.next();
+ if (document == null) {
+ break;
+ }
+ final String tableName = determineTable(document);
+ addObservation(tableName, document);
+ }
+ } finally {
+ documentSource.close();
+ }
+ }
+
+ @Override
+ public String getSchemaName() {
+ return _schemaName;
+ }
+
/**
* Determines which table a particular document should be mapped to.
*
- * @param map
+ * @param document
* @return
*/
- protected abstract String determineTable(Map<String, ?> map);
+ protected abstract String determineTable(Document document);
- public void addObservation(String table, Map<String, ?> values) {
+ public void addObservation(String table, Document document) {
final InferentialTableBuilder tableBuilder = getTableBuilder(table);
- tableBuilder.addObservation(values);
+ tableBuilder.addObservation(document);
}
public InferentialTableBuilder getTableBuilder(String table) {
@@ -82,11 +108,15 @@ public abstract class InferentialSchemaBuilder implements SchemaBuilder {
final Set<String> tableNames = new TreeSet<String>(_tableBuilders.keySet());
for (final String tableName : tableNames) {
- final MutableTable table = getTableBuilder(tableName).buildTable();
+ final MutableTable table = buildTable(getTableBuilder(tableName));
table.setSchema(schema);
schema.addTable(table);
}
return schema;
}
+
+ protected MutableTable buildTable(InferentialTableBuilder tableBuilder) {
+ return tableBuilder.buildTable();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/InferentialTableBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/InferentialTableBuilder.java b/core/src/main/java/org/apache/metamodel/schema/builder/InferentialTableBuilder.java
index 66c575b..c2ad490 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/InferentialTableBuilder.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/InferentialTableBuilder.java
@@ -25,6 +25,8 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.DocumentSource;
import org.apache.metamodel.schema.MutableColumn;
import org.apache.metamodel.schema.MutableTable;
@@ -45,8 +47,9 @@ public class InferentialTableBuilder implements TableBuilder {
_observationCounter = new AtomicInteger();
}
- public void addObservation(Map<String, ?> values) {
+ public void addObservation(Document document) {
_observationCounter.incrementAndGet();
+ final Map<String, ?> values = document.getValues();
final Set<? extends Entry<?, ?>> entries = values.entrySet();
for (final Entry<?, ?> entry : entries) {
final Object key = entry.getKey();
@@ -77,7 +80,7 @@ public class InferentialTableBuilder implements TableBuilder {
final Set<String> columnNames = new TreeSet<String>(_columnBuilders.keySet());
final MutableTable table = new MutableTable(_tableName);
- int columnNumber = 1;
+ int columnNumber = 0;
for (final String columnName : columnNames) {
final InferentialColumnBuilder columnBuilder = getColumnBuilder(columnName);
final MutableColumn column = columnBuilder.build();
@@ -112,7 +115,7 @@ public class InferentialTableBuilder implements TableBuilder {
@Override
public void offerSource(DocumentSource documentSource) {
while (getObservationCount() < MAX_SAMPLE_SIZE) {
- Map<String, ?> map = documentSource.next();
+ Document map = documentSource.next();
if (map == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/LazyDocumentSource.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/LazyDocumentSource.java b/core/src/main/java/org/apache/metamodel/schema/builder/LazyDocumentSource.java
index 3b432d6..d759372 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/LazyDocumentSource.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/LazyDocumentSource.java
@@ -18,8 +18,8 @@
*/
package org.apache.metamodel.schema.builder;
-import java.util.Map;
-
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.DocumentSource;
import org.apache.metamodel.util.LazyRef;
/**
@@ -37,7 +37,7 @@ public class LazyDocumentSource implements DocumentSource {
}
@Override
- public Map<String, ?> next() {
+ public Document next() {
return _lazyRef.get().next();
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/MultiTableInferentialSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/MultiTableInferentialSchemaBuilder.java b/core/src/main/java/org/apache/metamodel/schema/builder/MultiTableInferentialSchemaBuilder.java
index f0e7b4b..d577a05 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/MultiTableInferentialSchemaBuilder.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/MultiTableInferentialSchemaBuilder.java
@@ -18,36 +18,54 @@
*/
package org.apache.metamodel.schema.builder;
-import java.util.Map;
-
+import org.apache.metamodel.convert.DocumentConverter;
+import org.apache.metamodel.data.Document;
import org.apache.metamodel.schema.Table;
import org.apache.metamodel.util.Resource;
import org.apache.metamodel.util.ResourceUtils;
/**
- * {@link InferentialSchemaBuilder} that produces multiple table .
+ * {@link InferentialSchemaBuilder} that produces multiple tables based on a
+ * discriminator column - a column that contains the table name.
*/
public class MultiTableInferentialSchemaBuilder extends InferentialSchemaBuilder {
private final String _discriminatorColumn;
+ public MultiTableInferentialSchemaBuilder(Resource resource) {
+ this(ResourceUtils.getParentName(resource));
+ }
+
public MultiTableInferentialSchemaBuilder(Resource resource, String discriminatorColumn) {
this(ResourceUtils.getParentName(resource), discriminatorColumn);
}
+ public MultiTableInferentialSchemaBuilder(String schemaName) {
+ this(schemaName, null);
+ }
+
public MultiTableInferentialSchemaBuilder(String schemaName, String discriminatorColumn) {
super(schemaName);
_discriminatorColumn = discriminatorColumn;
}
-
@Override
- protected String determineTable(Map<String, ?> map) {
- Object value = map.get(_discriminatorColumn);
- if (value == null) {
+ protected String determineTable(Document document) {
+ final String tableName;
+ if (_discriminatorColumn == null) {
+ tableName = document.getSourceCollectionName();
+ } else {
+ final Object value = document.getValues().get(_discriminatorColumn);
+ if (value == null) {
+ tableName = null;
+ } else {
+ tableName = value.toString();
+ }
+ }
+ if (tableName == null) {
return "(other)";
}
- return value.toString();
+ return tableName;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/SchemaBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/SchemaBuilder.java b/core/src/main/java/org/apache/metamodel/schema/builder/SchemaBuilder.java
index 9ba2328..33f6016 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/SchemaBuilder.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/SchemaBuilder.java
@@ -18,6 +18,7 @@
*/
package org.apache.metamodel.schema.builder;
+import org.apache.metamodel.convert.DocumentConverter;
import org.apache.metamodel.schema.MutableSchema;
import org.apache.metamodel.schema.Schema;
import org.apache.metamodel.schema.Table;
@@ -28,14 +29,14 @@ import org.apache.metamodel.schema.Table;
public interface SchemaBuilder {
/**
- * Offers a {@link DocumentSource} to the {@link SchemaBuilder}. The
- * {@link SchemaBuilder} may consume the source to build/detect a schema
- * based on the observed documents in the source. It may also choose to
+ * Offers a {@link DocumentSourceProvider} to the {@link SchemaBuilder}. The
+ * {@link SchemaBuilder} may consume the sources to build/detect a schema
+ * based on the observed documents in the sources. It may also choose to
* ignore the source, if the it does not need it.
*
- * @param documentSource
+ * @param documentSourceProvider
*/
- public void offerSource(DocumentSource documentSource);
+ public void offerSources(DocumentSourceProvider documentSourceProvider);
/**
* Builds the {@link Schema}
@@ -50,4 +51,11 @@ public interface SchemaBuilder {
* @return
*/
public DocumentConverter getDocumentConverter(Table table);
+
+ /**
+ * Gets the name of the schema that is built / will be built.
+ *
+ * @return
+ */
+ public String getSchemaName();
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/SimpleTableDefSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/SimpleTableDefSchemaBuilder.java b/core/src/main/java/org/apache/metamodel/schema/builder/SimpleTableDefSchemaBuilder.java
index f59b74d..e2d3a63 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/SimpleTableDefSchemaBuilder.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/SimpleTableDefSchemaBuilder.java
@@ -18,6 +18,7 @@
*/
package org.apache.metamodel.schema.builder;
+import org.apache.metamodel.convert.DocumentConverter;
import org.apache.metamodel.schema.MutableSchema;
import org.apache.metamodel.schema.MutableTable;
import org.apache.metamodel.schema.Table;
@@ -38,9 +39,14 @@ public class SimpleTableDefSchemaBuilder implements SchemaBuilder {
}
@Override
- public void offerSource(DocumentSource documentSource) {
+ public void offerSources(DocumentSourceProvider documentSource) {
// do nothing
}
+
+ @Override
+ public String getSchemaName() {
+ return _schemaName;
+ }
@Override
public MutableSchema build() {
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/SingleMapColumnSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/SingleMapColumnSchemaBuilder.java b/core/src/main/java/org/apache/metamodel/schema/builder/SingleMapColumnSchemaBuilder.java
index ed46552..440f69e 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/SingleMapColumnSchemaBuilder.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/SingleMapColumnSchemaBuilder.java
@@ -20,8 +20,10 @@ package org.apache.metamodel.schema.builder;
import java.util.Map;
+import org.apache.metamodel.convert.DocumentConverter;
import org.apache.metamodel.data.DataSetHeader;
import org.apache.metamodel.data.DefaultRow;
+import org.apache.metamodel.data.Document;
import org.apache.metamodel.data.Row;
import org.apache.metamodel.schema.ColumnType;
import org.apache.metamodel.schema.MutableColumn;
@@ -52,9 +54,14 @@ public class SingleMapColumnSchemaBuilder implements SchemaBuilder, DocumentConv
}
@Override
- public void offerSource(DocumentSource documentSource) {
+ public void offerSources(DocumentSourceProvider documentSourceProvider) {
// do nothing
}
+
+ @Override
+ public String getSchemaName() {
+ return _schemaName;
+ }
@Override
public MutableSchema build() {
@@ -71,10 +78,10 @@ public class SingleMapColumnSchemaBuilder implements SchemaBuilder, DocumentConv
}
@Override
- public Row convert(Map<String, ?> document, DataSetHeader header) {
+ public Row convert(Document document, DataSetHeader header) {
assert header.size() == 1;
Object[] values = new Object[1];
- values[0] = document;
+ values[0] = document.getValues();
return new DefaultRow(header, values);
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/SingleTableInferentialSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/SingleTableInferentialSchemaBuilder.java b/core/src/main/java/org/apache/metamodel/schema/builder/SingleTableInferentialSchemaBuilder.java
index dee764a..0119fa5 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/SingleTableInferentialSchemaBuilder.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/SingleTableInferentialSchemaBuilder.java
@@ -18,8 +18,9 @@
*/
package org.apache.metamodel.schema.builder;
-import java.util.Map;
-
+import org.apache.metamodel.convert.DocumentConverter;
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.DocumentSource;
import org.apache.metamodel.schema.Table;
import org.apache.metamodel.util.Resource;
import org.apache.metamodel.util.ResourceUtils;
@@ -41,12 +42,13 @@ public class SingleTableInferentialSchemaBuilder extends InferentialSchemaBuilde
}
@Override
- public void offerSource(DocumentSource documentSource) {
+ public void offerSources(DocumentSourceProvider documentSourceProvider) {
+ final DocumentSource documentSource = documentSourceProvider.getMixedDocumentSourceForSampling();
getTableBuilder(_tableName).offerSource(documentSource);
}
@Override
- protected String determineTable(Map<String, ?> map) {
+ protected String determineTable(Document document) {
return _tableName;
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/core/src/main/java/org/apache/metamodel/schema/builder/TableBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/metamodel/schema/builder/TableBuilder.java b/core/src/main/java/org/apache/metamodel/schema/builder/TableBuilder.java
index 6612eac..b137c27 100644
--- a/core/src/main/java/org/apache/metamodel/schema/builder/TableBuilder.java
+++ b/core/src/main/java/org/apache/metamodel/schema/builder/TableBuilder.java
@@ -18,6 +18,7 @@
*/
package org.apache.metamodel.schema.builder;
+import org.apache.metamodel.data.DocumentSource;
import org.apache.metamodel.schema.Column;
import org.apache.metamodel.schema.MutableTable;
import org.apache.metamodel.schema.Table;
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java
----------------------------------------------------------------------
diff --git a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java
index 28eb2d4..3439c36 100644
--- a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java
+++ b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java
@@ -18,14 +18,7 @@
*/
package org.apache.metamodel.couchdb;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
import org.apache.metamodel.MetaModelException;
import org.apache.metamodel.MetaModelHelper;
@@ -33,45 +26,39 @@ import org.apache.metamodel.QueryPostprocessDataContext;
import org.apache.metamodel.UpdateScript;
import org.apache.metamodel.UpdateableDataContext;
import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DocumentSource;
import org.apache.metamodel.data.SimpleDataSetHeader;
import org.apache.metamodel.query.FilterItem;
import org.apache.metamodel.query.SelectItem;
import org.apache.metamodel.schema.Column;
-import org.apache.metamodel.schema.ColumnType;
-import org.apache.metamodel.schema.MutableSchema;
-import org.apache.metamodel.schema.MutableTable;
import org.apache.metamodel.schema.Schema;
import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.schema.builder.DocumentSourceProvider;
import org.apache.metamodel.schema.builder.SchemaBuilder;
-import org.apache.metamodel.schema.builder.SimpleTableDefSchemaBuilder;
import org.apache.metamodel.util.SimpleTableDef;
import org.codehaus.jackson.JsonNode;
import org.ektorp.CouchDbConnector;
import org.ektorp.CouchDbInstance;
-import org.ektorp.DbAccessException;
import org.ektorp.StreamingViewResult;
import org.ektorp.ViewQuery;
-import org.ektorp.ViewResult.Row;
import org.ektorp.http.HttpClient;
import org.ektorp.http.StdHttpClient;
import org.ektorp.impl.StdCouchDbInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* DataContext implementation for CouchDB
*/
-public class CouchDbDataContext extends QueryPostprocessDataContext implements UpdateableDataContext {
+public class CouchDbDataContext extends QueryPostprocessDataContext implements UpdateableDataContext,
+ DocumentSourceProvider {
- private static final Logger logger = LoggerFactory.getLogger(CouchDbDataContext.class);
+ public static final String SCHEMA_NAME = "CouchDB";
public static final int DEFAULT_PORT = 5984;
public static final String FIELD_ID = "_id";
public static final String FIELD_REV = "_rev";
- private static final String SCHEMA_NAME = "CouchDB";
-
+ // the instance represents a handle to the whole couchdb cluster
private final CouchDbInstance _couchDbInstance;
private final SchemaBuilder _schemaBuilder;
@@ -92,113 +79,18 @@ public class CouchDbDataContext extends QueryPostprocessDataContext implements U
}
public CouchDbDataContext(CouchDbInstance couchDbInstance) {
- this(couchDbInstance, detectSchema(couchDbInstance));
- }
-
- public CouchDbDataContext(CouchDbInstance couchDbInstance, SimpleTableDef... tableDefs) {
- // the instance represents a handle to the whole couchdb cluster
_couchDbInstance = couchDbInstance;
- _schemaBuilder = new SimpleTableDefSchemaBuilder(SCHEMA_NAME, tableDefs);
- }
-
- public static SimpleTableDef[] detectSchema(CouchDbInstance couchDbInstance) {
- final List<SimpleTableDef> tableDefs = new ArrayList<SimpleTableDef>();
- final List<String> databaseNames = couchDbInstance.getAllDatabases();
- for (final String databaseName : databaseNames) {
-
- if (databaseName.startsWith("_")) {
- // don't add system tables
- continue;
- }
-
- CouchDbConnector connector = couchDbInstance.createConnector(databaseName, false);
-
- SimpleTableDef tableDef = detectTable(connector);
- tableDefs.add(tableDef);
- }
- return tableDefs.toArray(new SimpleTableDef[tableDefs.size()]);
+ _schemaBuilder = new CouchDbInferentialSchemaBuilder();
}
- public static SimpleTableDef detectTable(CouchDbConnector connector) {
- final SortedMap<String, Set<ColumnType>> columnsAndTypes = new TreeMap<String, Set<ColumnType>>();
-
- final StreamingViewResult streamingView = connector.queryForStreamingView(new ViewQuery().allDocs()
- .includeDocs(true).limit(1000));
- try {
- final Iterator<Row> rowIterator = streamingView.iterator();
- while (safeHasNext(rowIterator)) {
- Row row = rowIterator.next();
- JsonNode doc = row.getDocAsNode();
-
- final Iterator<Entry<String, JsonNode>> fieldIterator = doc.getFields();
- while (fieldIterator.hasNext()) {
- Entry<String, JsonNode> entry = fieldIterator.next();
- String key = entry.getKey();
-
- Set<ColumnType> types = columnsAndTypes.get(key);
-
- if (types == null) {
- types = new HashSet<ColumnType>();
- columnsAndTypes.put(key, types);
- }
-
- JsonNode value = entry.getValue();
- if (value == null || value.isNull()) {
- // do nothing
- } else if (value.isTextual()) {
- types.add(ColumnType.VARCHAR);
- } else if (value.isArray()) {
- types.add(ColumnType.LIST);
- } else if (value.isObject()) {
- types.add(ColumnType.MAP);
- } else if (value.isBoolean()) {
- types.add(ColumnType.BOOLEAN);
- } else if (value.isInt()) {
- types.add(ColumnType.INTEGER);
- } else if (value.isLong()) {
- types.add(ColumnType.BIGINT);
- } else if (value.isDouble()) {
- types.add(ColumnType.DOUBLE);
- }
- }
-
- }
- } finally {
- streamingView.close();
- }
-
- final String[] columnNames = new String[columnsAndTypes.size()];
- final ColumnType[] columnTypes = new ColumnType[columnsAndTypes.size()];
-
- int i = 0;
- for (Entry<String, Set<ColumnType>> columnAndTypes : columnsAndTypes.entrySet()) {
- final String columnName = columnAndTypes.getKey();
- final Set<ColumnType> columnTypeSet = columnAndTypes.getValue();
- final ColumnType columnType;
- if (columnTypeSet.isEmpty()) {
- columnType = ColumnType.OTHER;
- } else if (columnTypeSet.size() == 1) {
- columnType = columnTypeSet.iterator().next();
- } else {
- // TODO: Select best type?
- columnType = ColumnType.OTHER;
- }
- columnNames[i] = columnName;
- columnTypes[i] = columnType;
- i++;
- }
-
- final SimpleTableDef tableDef = new SimpleTableDef(connector.getDatabaseName(), columnNames, columnTypes);
- return tableDef;
+ public CouchDbDataContext(CouchDbInstance couchDbInstance, String... databaseNames) {
+ _couchDbInstance = couchDbInstance;
+ _schemaBuilder = new CouchDbInferentialSchemaBuilder(databaseNames);
}
- private static boolean safeHasNext(Iterator<Row> rowIterator) {
- try {
- return rowIterator.hasNext();
- } catch (DbAccessException e) {
- logger.warn("Failed to move to next row while detecting table", e);
- return false;
- }
+ public CouchDbDataContext(CouchDbInstance couchDbInstance, SimpleTableDef... tableDefs) {
+ _couchDbInstance = couchDbInstance;
+ _schemaBuilder = new CouchDbSimpleTableDefSchemaBuilder(tableDefs);
}
public CouchDbInstance getCouchDbInstance() {
@@ -207,17 +99,13 @@ public class CouchDbDataContext extends QueryPostprocessDataContext implements U
@Override
protected Schema getMainSchema() throws MetaModelException {
- final MutableSchema schema = _schemaBuilder.build();
- final MutableTable[] tables = schema.getTables();
- for (MutableTable table : tables) {
- CouchDbTableCreationBuilder.addMandatoryColumns(table);
- }
- return schema;
+ _schemaBuilder.offerSources(this);
+ return _schemaBuilder.build();
}
@Override
protected String getMainSchemaName() throws MetaModelException {
- return SCHEMA_NAME;
+ return _schemaBuilder.getSchemaName();
}
@Override
@@ -286,4 +174,14 @@ public class CouchDbDataContext extends QueryPostprocessDataContext implements U
callback.close();
}
}
+
+ @Override
+ public DocumentSource getMixedDocumentSourceForSampling() {
+ return new CouchDbSamplingDocumentSource(_couchDbInstance);
+ }
+
+ @Override
+ public DocumentSource getDocumentSourceForTable(String sourceCollectionName) {
+ return new CouchDbDatabaseDocumentSource(_couchDbInstance, sourceCollectionName, -1);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDatabaseDocumentSource.java
----------------------------------------------------------------------
diff --git a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDatabaseDocumentSource.java b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDatabaseDocumentSource.java
new file mode 100644
index 0000000..6a11c1e
--- /dev/null
+++ b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDatabaseDocumentSource.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.couchdb;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.DocumentSource;
+import org.codehaus.jackson.JsonNode;
+import org.ektorp.CouchDbConnector;
+import org.ektorp.CouchDbInstance;
+import org.ektorp.StreamingViewResult;
+import org.ektorp.ViewQuery;
+import org.ektorp.ViewResult.Row;
+
+/**
+ * {@link DocumentSource} implementation that goes with the
+ * {@link CouchDbDataContext}
+ */
+final class CouchDbDatabaseDocumentSource implements DocumentSource {
+
+ private final CouchDbInstance _couchDbInstance;
+ private final StreamingViewResult _view;
+ private final String _databaseName;
+ private final Iterator<Row> _rowIterator;
+ private final AtomicBoolean _closed;
+
+ public CouchDbDatabaseDocumentSource(CouchDbInstance couchDbInstance, String databaseName, int maxRows) {
+ _couchDbInstance = couchDbInstance;
+ _databaseName = databaseName;
+ _closed = new AtomicBoolean(false);
+
+ final CouchDbConnector tableConnector = _couchDbInstance.createConnector(databaseName, false);
+ if (maxRows > -1) {
+ _view = tableConnector.queryForStreamingView(new ViewQuery().allDocs().includeDocs(true).limit(maxRows));
+ } else {
+ _view = tableConnector.queryForStreamingView(new ViewQuery().allDocs().includeDocs(true));
+ }
+ _rowIterator = _view.iterator();
+ }
+
+ @Override
+ public Document next() {
+ if (_closed.get()) {
+ return null;
+ }
+
+ if (!CouchDbUtils.safeHasNext(_rowIterator)) {
+ close();
+ return next();
+ }
+
+ final Row row = _rowIterator.next();
+ final JsonNode docNode = row.getDocAsNode();
+ final Map<String, Object> map = CouchDbUtils.jsonNodeToMap(docNode);
+ return new Document(_databaseName, map, row);
+ }
+
+ @Override
+ public void close() {
+ boolean closedBefore = _closed.getAndSet(true);
+ if (!closedBefore) {
+ _view.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDocumentConverter.java
----------------------------------------------------------------------
diff --git a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDocumentConverter.java b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDocumentConverter.java
new file mode 100644
index 0000000..e1277e7
--- /dev/null
+++ b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDocumentConverter.java
@@ -0,0 +1,18 @@
+package org.apache.metamodel.couchdb;
+
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.schema.builder.ColumnNameAsKeysRowConverter;
+import org.ektorp.ViewResult.Row;
+
+public class CouchDbDocumentConverter extends ColumnNameAsKeysRowConverter {
+
+ @Override
+ protected Object get(Document document, String columnName) {
+ if (CouchDbDataContext.FIELD_ID.equals(columnName)) {
+ Row row = (Row) document.getSourceObject();
+ return row.getId();
+ }
+ // TODO Auto-generated method stub
+ return super.get(document, columnName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbInferentialSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbInferentialSchemaBuilder.java b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbInferentialSchemaBuilder.java
new file mode 100644
index 0000000..a320451
--- /dev/null
+++ b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbInferentialSchemaBuilder.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.couchdb;
+
+import org.apache.metamodel.convert.DocumentConverter;
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.DocumentSource;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.schema.builder.DocumentSourceProvider;
+import org.apache.metamodel.schema.builder.InferentialSchemaBuilder;
+import org.apache.metamodel.schema.builder.InferentialTableBuilder;
+
+final class CouchDbInferentialSchemaBuilder extends InferentialSchemaBuilder {
+
+ private final String[] _databaseNames;
+
+ public CouchDbInferentialSchemaBuilder() {
+ this(null);
+ }
+
+ public CouchDbInferentialSchemaBuilder(String[] databaseNames) {
+ super(CouchDbDataContext.SCHEMA_NAME);
+ _databaseNames = databaseNames;
+ }
+
+ @Override
+ public void offerSources(DocumentSourceProvider documentSourceProvider) {
+ if (_databaseNames == null) {
+ super.offerSources(documentSourceProvider);
+ } else {
+ for (String databaseName : _databaseNames) {
+ final DocumentSource documentSource = documentSourceProvider.getDocumentSourceForTable(databaseName);
+ offerDocumentSource(documentSource);
+ }
+ }
+ }
+
+ @Override
+ public DocumentConverter getDocumentConverter(Table table) {
+ return new CouchDbDocumentConverter();
+ }
+
+ @Override
+ protected String determineTable(Document document) {
+ final String sourceCollectionName = document.getSourceCollectionName();
+ return sourceCollectionName;
+ }
+
+ @Override
+ protected MutableTable buildTable(InferentialTableBuilder tableBuilder) {
+ final MutableTable table = super.buildTable(tableBuilder);
+ CouchDbTableCreationBuilder.addMandatoryColumns(table);
+ return table;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbSamplingDocumentSource.java
----------------------------------------------------------------------
diff --git a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbSamplingDocumentSource.java b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbSamplingDocumentSource.java
new file mode 100644
index 0000000..623376c
--- /dev/null
+++ b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbSamplingDocumentSource.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.couchdb;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.DocumentSource;
+import org.ektorp.CouchDbInstance;
+
+/**
+ * A {@link DocumentSource} that instantiates several other
+ * {@link CouchDbDatabaseDocumentSource} on-demand when they are need to provide
+ * a cross-database sample
+ */
+public class CouchDbSamplingDocumentSource implements DocumentSource {
+
+ private final CouchDbInstance _couchDbInstance;
+ private final Iterator<String> _databaseNameIterator;
+ private volatile CouchDbDatabaseDocumentSource _currentSource;
+
+ public CouchDbSamplingDocumentSource(CouchDbInstance couchDbInstance) {
+ _couchDbInstance = couchDbInstance;
+ final List<String> allDatabases = _couchDbInstance.getAllDatabases();
+ _databaseNameIterator = allDatabases.iterator();
+ _currentSource = null;
+ }
+
+ @Override
+ public Document next() {
+ if (_currentSource == null) {
+ if (!_databaseNameIterator.hasNext()) {
+ return null;
+ }
+ String databaseName = _databaseNameIterator.next();
+ while (databaseName.startsWith("_")) {
+ if (!_databaseNameIterator.hasNext()) {
+ return null;
+ }
+ databaseName = _databaseNameIterator.next();
+ }
+ _currentSource = new CouchDbDatabaseDocumentSource(_couchDbInstance, databaseName, 500);
+ }
+
+ final Document next = _currentSource.next();
+ if (next == null) {
+ if (_currentSource != null) {
+ _currentSource.close();
+ }
+ _currentSource = null;
+ return next();
+ }
+
+ return next;
+ }
+
+ @Override
+ public void close() {
+ if (_currentSource != null) {
+ _currentSource.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbSimpleTableDefSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbSimpleTableDefSchemaBuilder.java b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbSimpleTableDefSchemaBuilder.java
new file mode 100644
index 0000000..308d8f2
--- /dev/null
+++ b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbSimpleTableDefSchemaBuilder.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.couchdb;
+
+import org.apache.metamodel.convert.DocumentConverter;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.schema.builder.DocumentSourceProvider;
+import org.apache.metamodel.schema.builder.SimpleTableDefSchemaBuilder;
+import org.apache.metamodel.util.SimpleTableDef;
+
+public class CouchDbSimpleTableDefSchemaBuilder extends SimpleTableDefSchemaBuilder {
+
+ public CouchDbSimpleTableDefSchemaBuilder(SimpleTableDef[] tableDefs) {
+ super(CouchDbDataContext.SCHEMA_NAME, tableDefs);
+ }
+
+ @Override
+ public void offerSources(DocumentSourceProvider documentSourceProvider) {
+
+ }
+
+ @Override
+ public MutableSchema build() {
+ MutableSchema schema = super.build();
+ MutableTable[] tables = schema.getTables();
+ for (MutableTable table : tables) {
+ CouchDbTableCreationBuilder.addMandatoryColumns(table);
+ }
+ return schema;
+ }
+
+ @Override
+ public DocumentConverter getDocumentConverter(Table table) {
+ return new CouchDbDocumentConverter();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbTableCreationBuilder.java
----------------------------------------------------------------------
diff --git a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbTableCreationBuilder.java b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbTableCreationBuilder.java
index b87d095..d81f768 100644
--- a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbTableCreationBuilder.java
+++ b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbTableCreationBuilder.java
@@ -18,7 +18,6 @@
*/
package org.apache.metamodel.couchdb;
-import org.ektorp.CouchDbInstance;
import org.apache.metamodel.MetaModelException;
import org.apache.metamodel.create.AbstractTableCreationBuilder;
import org.apache.metamodel.schema.ColumnType;
@@ -27,6 +26,7 @@ import org.apache.metamodel.schema.MutableSchema;
import org.apache.metamodel.schema.MutableTable;
import org.apache.metamodel.schema.Schema;
import org.apache.metamodel.schema.Table;
+import org.ektorp.CouchDbInstance;
final class CouchDbTableCreationBuilder extends AbstractTableCreationBuilder<CouchDbUpdateCallback> {
@@ -65,10 +65,17 @@ final class CouchDbTableCreationBuilder extends AbstractTableCreationBuilder<Cou
table.addColumn(0, idColumn);
}
idColumn.setPrimaryKey(true);
+ idColumn.setNullable(false);
}
- if (table.getColumnByName(CouchDbDataContext.FIELD_REV) == null) {
- table.addColumn(1, new MutableColumn(CouchDbDataContext.FIELD_REV, ColumnType.VARCHAR, table, 1, false));
+ // add or correct _rev column
+ {
+ MutableColumn revColumn = (MutableColumn) table.getColumnByName(CouchDbDataContext.FIELD_REV);
+ if (revColumn == null) {
+ revColumn = new MutableColumn(CouchDbDataContext.FIELD_REV, ColumnType.VARCHAR, table, 1, false);
+ table.addColumn(1, revColumn);
+ }
+ revColumn.setNullable(false);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbUtils.java
----------------------------------------------------------------------
diff --git a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbUtils.java b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbUtils.java
index 1708b28..c278e16 100644
--- a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbUtils.java
+++ b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.metamodel.couchdb;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -26,11 +27,27 @@ import org.apache.metamodel.data.DefaultRow;
import org.apache.metamodel.data.Row;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
+import org.ektorp.DbAccessException;
/**
* Convenience and utility methods for MetaModel's CouchDB adaptor
*/
final class CouchDbUtils {
+
+ /**
+ * Safely calls hasNext on a row iterator
+ *
+ * @param rowIterator
+ * @return
+ */
+ public static boolean safeHasNext(Iterator<?> rowIterator) {
+ try {
+ return rowIterator.hasNext();
+ } catch (DbAccessException e) {
+ return false;
+ }
+ }
+
/**
* Converts {@link JsonNode} to MetaModel {@link Row}.
*
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java
----------------------------------------------------------------------
diff --git a/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java b/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java
index d9e361e..4a89a6e 100644
--- a/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java
+++ b/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java
@@ -135,22 +135,45 @@ public class CouchDbDataContextTest extends CouchDbTestCase {
return;
}
+ final String databaseName = getDatabaseName();
+
+ {
+ // insert a document to provide data to do inferential schema
+ // detection
+ final CouchDbConnector connector = couchDbInstance.createConnector(databaseName, false);
+ final Map<String, Object> map = new HashMap<String, Object>();
+ map.put("foo", "bar");
+ map.put("bar", "baz");
+ map.put("baz", 1234);
+ connector.addToBulkBuffer(map);
+ connector.flushBulkBuffer();
+ }
+
final CouchDbDataContext dc = new CouchDbDataContext(couchDbInstance);
+ Table table = dc.getDefaultSchema().getTableByName(databaseName);
+ assertNotNull(table);
+ assertEquals("[Column[name=_id,columnNumber=0,type=VARCHAR,nullable=false,nativeType=null,columnSize=null], "
+ + "Column[name=_rev,columnNumber=1,type=VARCHAR,nullable=false,nativeType=null,columnSize=null], "
+ + "Column[name=bar,columnNumber=2,type=VARCHAR,nullable=null,nativeType=null,columnSize=null], "
+ + "Column[name=baz,columnNumber=3,type=INTEGER,nullable=null,nativeType=null,columnSize=null], "
+ + "Column[name=foo,columnNumber=4,type=VARCHAR,nullable=null,nativeType=null,columnSize=null]]",
+ Arrays.toString(table.getColumns()));
// first delete the manually created database!
dc.executeUpdate(new UpdateScript() {
@Override
public void run(UpdateCallback callback) {
- callback.dropTable(getDatabaseName()).execute();
+ callback.dropTable(databaseName).execute();
}
});
- assertNull(dc.getDefaultSchema().getTableByName(getDatabaseName()));
+ table = dc.getDefaultSchema().getTableByName(databaseName);
+ assertNull(table);
dc.executeUpdate(new UpdateScript() {
@Override
public void run(UpdateCallback callback) {
- Table table = callback.createTable(dc.getDefaultSchema(), getDatabaseName()).withColumn("foo")
+ Table table = callback.createTable(dc.getDefaultSchema(), databaseName).withColumn("foo")
.ofType(ColumnType.VARCHAR).withColumn("greeting").ofType(ColumnType.VARCHAR).execute();
assertEquals("[_id, _rev, foo, greeting]", Arrays.toString(table.getColumnNames()));
}
@@ -159,12 +182,12 @@ public class CouchDbDataContextTest extends CouchDbTestCase {
dc.executeUpdate(new UpdateScript() {
@Override
public void run(UpdateCallback callback) {
- callback.insertInto(getDatabaseName()).value("foo", "bar").value("greeting", "hello").execute();
- callback.insertInto(getDatabaseName()).value("foo", "baz").value("greeting", "hi").execute();
+ callback.insertInto(databaseName).value("foo", "bar").value("greeting", "hello").execute();
+ callback.insertInto(databaseName).value("foo", "baz").value("greeting", "hi").execute();
}
});
- DataSet ds = dc.query().from(getDatabaseName()).select("_id", "foo", "greeting").execute();
+ DataSet ds = dc.query().from(databaseName).select("_id", "foo", "greeting").execute();
assertTrue(ds.next());
assertNotNull(ds.getRow().getValue(0));
assertEquals("bar", ds.getRow().getValue(1));
@@ -179,13 +202,13 @@ public class CouchDbDataContextTest extends CouchDbTestCase {
dc.executeUpdate(new UpdateScript() {
@Override
public void run(UpdateCallback callback) {
- callback.update(getDatabaseName()).value("greeting", "howdy").where("foo").isEquals("baz").execute();
+ callback.update(databaseName).value("greeting", "howdy").where("foo").isEquals("baz").execute();
- callback.update(getDatabaseName()).value("foo", "foo").where("foo").isEquals("bar").execute();
+ callback.update(databaseName).value("foo", "foo").where("foo").isEquals("bar").execute();
}
});
- ds = dc.query().from(getDatabaseName()).select("_id", "foo", "greeting").execute();
+ ds = dc.query().from(databaseName).select("_id", "foo", "greeting").execute();
assertTrue(ds.next());
assertNotNull(ds.getRow().getValue(0));
assertEquals("foo", ds.getRow().getValue(1));
@@ -220,8 +243,7 @@ public class CouchDbDataContextTest extends CouchDbTestCase {
}
// create datacontext using detected schema
- SimpleTableDef tableDef = CouchDbDataContext.detectTable(connector);
- CouchDbDataContext dc = new CouchDbDataContext(couchDbInstance, tableDef);
+ CouchDbDataContext dc = new CouchDbDataContext(couchDbInstance, getDatabaseName());
// verify schema and execute query
Schema schema = dc.getMainSchema();
@@ -230,11 +252,11 @@ public class CouchDbDataContextTest extends CouchDbTestCase {
assertEquals("[_id, _rev, age, gender, name]",
Arrays.toString(schema.getTableByName(getDatabaseName()).getColumnNames()));
Column idColumn = schema.getTableByName(getDatabaseName()).getColumnByName("_id");
- assertEquals("Column[name=_id,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]",
+ assertEquals("Column[name=_id,columnNumber=0,type=VARCHAR,nullable=false,nativeType=null,columnSize=null]",
idColumn.toString());
assertTrue(idColumn.isPrimaryKey());
- assertEquals("Column[name=_rev,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]",
+ assertEquals("Column[name=_rev,columnNumber=1,type=VARCHAR,nullable=false,nativeType=null,columnSize=null]",
schema.getTableByName(getDatabaseName()).getColumnByName("_rev").toString());
DataSet ds;
@@ -300,11 +322,11 @@ public class CouchDbDataContextTest extends CouchDbTestCase {
}
// create datacontext using detected schema
- SimpleTableDef tableDef = CouchDbDataContext.detectTable(connector);
- CouchDbDataContext dc = new CouchDbDataContext(couchDbInstance, tableDef);
+ final String databaseName = getDatabaseName();
+ final CouchDbDataContext dc = new CouchDbDataContext(couchDbInstance, databaseName);
- DataSet ds1 = dc.query().from(getDatabaseName()).select("name").and("age").firstRow(2).execute();
- DataSet ds2 = dc.query().from(getDatabaseName()).select("name").and("age").maxRows(1).execute();
+ final DataSet ds1 = dc.query().from(databaseName).select("name").and("age").firstRow(2).execute();
+ final DataSet ds2 = dc.query().from(databaseName).select("name").and("age").maxRows(1).execute();
assertTrue("Class: " + ds1.getClass().getName(), ds1 instanceof CouchDbDataSet);
assertTrue("Class: " + ds2.getClass().getName(), ds2 instanceof CouchDbDataSet);
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/json/src/main/java/org/apache/metamodel/json/JsonDataContext.java
----------------------------------------------------------------------
diff --git a/json/src/main/java/org/apache/metamodel/json/JsonDataContext.java b/json/src/main/java/org/apache/metamodel/json/JsonDataContext.java
index aa22da6..1700f93 100644
--- a/json/src/main/java/org/apache/metamodel/json/JsonDataContext.java
+++ b/json/src/main/java/org/apache/metamodel/json/JsonDataContext.java
@@ -25,24 +25,24 @@ import org.apache.metamodel.DataContext;
import org.apache.metamodel.MetaModelException;
import org.apache.metamodel.MetaModelHelper;
import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.convert.DocumentConverter;
import org.apache.metamodel.data.CachingDataSetHeader;
import org.apache.metamodel.data.DataSet;
import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.DocumentSource;
+import org.apache.metamodel.data.DocumentSourceDataSet;
import org.apache.metamodel.data.MaxRowsDataSet;
+import org.apache.metamodel.data.MaxRowsDocumentSource;
import org.apache.metamodel.query.SelectItem;
import org.apache.metamodel.schema.Column;
import org.apache.metamodel.schema.Schema;
import org.apache.metamodel.schema.Table;
-import org.apache.metamodel.schema.builder.DocumentConverter;
-import org.apache.metamodel.schema.builder.DocumentSource;
-import org.apache.metamodel.schema.builder.LazyDocumentSource;
+import org.apache.metamodel.schema.builder.DocumentSourceProvider;
import org.apache.metamodel.schema.builder.SchemaBuilder;
import org.apache.metamodel.schema.builder.SingleTableInferentialSchemaBuilder;
import org.apache.metamodel.util.FileHelper;
import org.apache.metamodel.util.FileResource;
-import org.apache.metamodel.util.LazyRef;
import org.apache.metamodel.util.Resource;
-import org.apache.metamodel.util.ResourceUtils;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.map.MappingJsonFactory;
import org.slf4j.Logger;
@@ -52,7 +52,7 @@ import org.slf4j.LoggerFactory;
* {@link DataContext} implementation that works on JSON files or
* {@link Resource}s.
*/
-public class JsonDataContext extends QueryPostprocessDataContext {
+public class JsonDataContext extends QueryPostprocessDataContext implements DocumentSourceProvider {
private static final Logger logger = LoggerFactory.getLogger(JsonDataContext.class);
@@ -74,35 +74,23 @@ public class JsonDataContext extends QueryPostprocessDataContext {
@Override
protected Schema getMainSchema() throws MetaModelException {
- final LazyDocumentSource documentSource = new LazyDocumentSource(new LazyRef<DocumentSource>() {
- @Override
- protected DocumentSource fetch() throws Throwable {
- final JsonDocumentSource jsonDocumentIterator = createJsonDocumentIterator();
- return jsonDocumentIterator;
- }
- });
- try {
- _schemaBuilder.offerSource(documentSource);
- return _schemaBuilder.build();
- } finally {
- documentSource.close();
- }
+ _schemaBuilder.offerSources(this);
+ return _schemaBuilder.build();
}
@Override
protected String getMainSchemaName() throws MetaModelException {
- return ResourceUtils.getParentName(_resource);
+ return _schemaBuilder.getSchemaName();
}
@Override
protected DataSet materializeMainSchemaTable(Table table, Column[] columns, int maxRows) {
-
final DocumentConverter documentConverter = _schemaBuilder.getDocumentConverter(table);
final SelectItem[] selectItems = MetaModelHelper.createSelectItems(columns);
final DataSetHeader header = new CachingDataSetHeader(selectItems);
- final JsonDocumentSource documentSource = createJsonDocumentIterator();
+ final DocumentSource documentSource = getDocumentSourceForTable(table.getName());
- DataSet dataSet = new JsonDataSet(header, documentSource, documentConverter);
+ DataSet dataSet = new DocumentSourceDataSet(header, documentSource, documentConverter);
if (maxRows > 0) {
dataSet = new MaxRowsDataSet(dataSet, maxRows);
@@ -111,17 +99,28 @@ public class JsonDataContext extends QueryPostprocessDataContext {
return dataSet;
}
- private JsonDocumentSource createJsonDocumentIterator() {
+ private DocumentSource createDocumentSource() {
final InputStream inputStream = _resource.read();
try {
final MappingJsonFactory jsonFactory = new MappingJsonFactory();
final JsonParser parser = jsonFactory.createJsonParser(inputStream);
logger.debug("Created JSON parser for resource: {}", _resource);
- return new JsonDocumentSource(parser);
+ return new JsonDocumentSource(parser, _resource.getName());
} catch (Exception e) {
FileHelper.safeClose(inputStream);
throw new MetaModelException("Unexpected error while creating JSON parser", e);
}
}
+
+ @Override
+ public DocumentSource getMixedDocumentSourceForSampling() {
+ return new MaxRowsDocumentSource(createDocumentSource(), 1000);
+ }
+
+ @Override
+ public DocumentSource getDocumentSourceForTable(String sourceCollectionName) {
+ // only a single source collection - returning that
+ return createDocumentSource();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/json/src/main/java/org/apache/metamodel/json/JsonDataSet.java
----------------------------------------------------------------------
diff --git a/json/src/main/java/org/apache/metamodel/json/JsonDataSet.java b/json/src/main/java/org/apache/metamodel/json/JsonDataSet.java
deleted file mode 100644
index 5aaae74..0000000
--- a/json/src/main/java/org/apache/metamodel/json/JsonDataSet.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.metamodel.json;
-
-import java.util.Map;
-
-import org.apache.metamodel.data.AbstractDataSet;
-import org.apache.metamodel.data.DataSetHeader;
-import org.apache.metamodel.data.Row;
-import org.apache.metamodel.schema.builder.DocumentConverter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-final class JsonDataSet extends AbstractDataSet {
-
- private static final Logger logger = LoggerFactory.getLogger(JsonDataSet.class);
-
- private final JsonDocumentSource _jsonDocumentSource;
- private final DocumentConverter _converter;
- private volatile Map<String, ?> _map;
-
- public JsonDataSet(DataSetHeader header, JsonDocumentSource jsonDocumentSource, DocumentConverter converter) {
- super(header);
- _jsonDocumentSource = jsonDocumentSource;
- _converter = converter;
- }
-
- @Override
- public boolean next() {
- _map = _jsonDocumentSource.next();
- if (_map == null) {
- return false;
- }
- return true;
- }
-
- @Override
- public Row getRow() {
- if (_map == null) {
- return null;
- }
- final DataSetHeader header = getHeader();
- return _converter.convert(_map, header);
- }
-
- @Override
- public void close() {
- super.close();
- try {
- _jsonDocumentSource.close();
- } catch (Exception e) {
- logger.warn("Failed to close JSON parser", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/b4c6a77b/json/src/main/java/org/apache/metamodel/json/JsonDocumentSource.java
----------------------------------------------------------------------
diff --git a/json/src/main/java/org/apache/metamodel/json/JsonDocumentSource.java b/json/src/main/java/org/apache/metamodel/json/JsonDocumentSource.java
index 1ac3f5c..fe2b9a9 100644
--- a/json/src/main/java/org/apache/metamodel/json/JsonDocumentSource.java
+++ b/json/src/main/java/org/apache/metamodel/json/JsonDocumentSource.java
@@ -22,7 +22,8 @@ import java.io.IOException;
import java.util.Map;
import org.apache.metamodel.MetaModelException;
-import org.apache.metamodel.schema.builder.DocumentSource;
+import org.apache.metamodel.data.Document;
+import org.apache.metamodel.data.DocumentSource;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonToken;
import org.slf4j.Logger;
@@ -34,14 +35,16 @@ import org.slf4j.LoggerFactory;
final class JsonDocumentSource implements DocumentSource {
private static final Logger logger = LoggerFactory.getLogger(JsonDocumentSource.class);
-
+
private final JsonParser _parser;
+ private final String _sourceCollectionName;
- public JsonDocumentSource(JsonParser parser) {
+ public JsonDocumentSource(JsonParser parser, String sourceCollectionName) {
_parser = parser;
+ _sourceCollectionName = sourceCollectionName;
}
- public Map<String, ?> next() {
+ public Document next() {
while (true) {
final JsonToken token = getNextToken();
if (token == null) {
@@ -49,7 +52,8 @@ final class JsonDocumentSource implements DocumentSource {
}
if (token == JsonToken.START_OBJECT) {
- return readValue();
+ Map<String, ?> value = readValue();
+ return new Document(_sourceCollectionName, value, value);
}
}
}