You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@metamodel.apache.org by ashish-m-yh <gi...@git.apache.org> on 2015/04/26 19:35:14 UTC
[GitHub] metamodel pull request: Fresh PR for Sor
GitHub user ashish-m-yh opened a pull request:
https://github.com/apache/metamodel/pull/19
Fresh PR for Sor
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/ashish-m-yh/metamodel master
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/metamodel/pull/19.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #19
----
commit 752aaea9f05148ab76f942f675319a5fe05bb5d7
Author: Ashish Mukherjee <as...@gmail.com>
Date: 2015-04-17T18:55:10Z
Solr connector added
commit 77cd4f2ab5eb1452bcc52d08f5768681c8b021f8
Author: Ashish Mukherjee <as...@gmail.com>
Date: 2015-04-17T18:55:50Z
Additions for Solr connector
commit a25b5fc7352fd0829dfd45814f3803bfbc418d22
Author: Ashish Mukherjee <as...@gmail.com>
Date: 2015-04-17T18:56:06Z
Dependency added for Solr connector
commit 85fe92faf6b95ac9a94566939f63afb4ddcd50fc
Author: Ashish Mukherjee <as...@gmail.com>
Date: 2015-04-17T18:56:46Z
Lines added for Solr connector
commit 02b62d42966fe373b3c9ac26bef8baa7bad2b630
Author: Ashish Mukherjee <as...@gmail.com>
Date: 2015-04-20T18:44:04Z
Formatting changes and fixes based on PR comments
commit 0ac41ae258cf33d4a7a937238b0fa157eaf64640
Author: Ashish Mukherjee <as...@gmail.com>
Date: 2015-04-26T16:45:46Z
Simplified code for filter handling
commit 024087dc6189335be385a9578b2b0e8b29a8cd92
Author: Ashish Mukherjee <as...@gmail.com>
Date: 2015-04-26T16:47:52Z
Indentation fixes
commit 3e497a2d02758d6e6372871e3e56ca93c8b413c7
Author: Ashish Mukherjee <as...@gmail.com>
Date: 2015-04-26T17:33:08Z
Typo fixed
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29112571
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for ElasticSearch analytics engine.
--- End diff --
Too much copy/paste here :) I strongly assume the javadoc for ElasticSearch does not apply here.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29445084
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for ElasticSearch analytics engine.
+ *
+ * ElasticSearch has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ public static final String FIELD_ID = "id";
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum queryTypes {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ final Column idColumn = table.getColumnByName(FIELD_ID);
+ if (idColumn != null && idColumn instanceof MutableColumn) {
+ final MutableColumn mutableColumn = (MutableColumn) idColumn;
+ mutableColumn.setPrimaryKey(true);
+ }
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
--- End diff --
If I do not override executeQuery(...), then the Solr index cannot be queried by SQL, right? Also, I am calling super.executeQuery(q) at the end, to handle any other kind of query beyond the specific clauses I am handling. If the query is totally invalid and goes beyond any query which can be handled, exception may be thrown or DataSet null returned.
If you feel this approach still has some caveats, what method should be overridden so that the SQL interface still works?
What's your take?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-169699144
I think so. I'm sorry @ashish-m-yh but for me this PR is just a bit too far from finished to give anymore constructive feedback on. I suggest to spend a little more time on reviewing similar modules such as the ElasticSearch module, MongoDB, CouchDB or other modules - and then implement in a similar fashion.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569243
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
--- End diff --
The schema is referred to in the Query, it might not be the main schema - it might be the information schema.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29597584
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
--- End diff --
Joins are invalid on Solr
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29112490
--- Diff: solr/src/test/java/org/apache/metamodel/solr/SolrDataContextTest1.java ---
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.swing.table.TableModel;
+
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetTableModel;
+import org.apache.metamodel.data.FilteredDataSet;
+import org.apache.metamodel.data.InMemoryDataSet;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.Table;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class SolrDataContextTest1 {
--- End diff --
This class is somehow not running as part of continuous build. I think it is because the class name ends on 1 ...
And if we enable it, I am sure it fails, which relates to my previous remark ... The tests should be able to run on any machine. But externalizing the connection info is valid, like we do it in SalesforceDataContextTest and others. Here we have properties that are optional in the __metamodel-integrationtest-configuration.properties__ file in your user home. If the properties are not specified, then we programmatically ensure that the test is skipped. That means that anyone with a Solr instance can configure the test to run without any source code modifications.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29112457
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataSet.java ---
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.List;
+import java.util.Set;
+import java.util.IdentityHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.metamodel.data.AbstractDataSet;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.DefaultRow;
+import org.apache.metamodel.schema.Column;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.client.solrj.response.FacetField.Count;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+
+/**
+ * {@link DataSet} implementation for ElasticSearch
+ */
+final class SolrDataSet extends AbstractDataSet {
+
+ private static final Logger logger = LoggerFactory
--- End diff --
Indentation seems to still be tabs instead of 4 spaces
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569223
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for ElasticSearch analytics engine.
+ *
+ * ElasticSearch has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ public static final String FIELD_ID = "id";
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum queryTypes {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ final Column idColumn = table.getColumnByName(FIELD_ID);
+ if (idColumn != null && idColumn instanceof MutableColumn) {
+ final MutableColumn mutableColumn = (MutableColumn) idColumn;
+ mutableColumn.setPrimaryKey(true);
+ }
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
--- End diff --
The SQL interface works well if you do not override this method at all. The method is already implemented in the superclass and will delegate to some of the protected methods that you can choose to override. It is also possible to override this method, but it comes at a much greater risk becase the Query object is so multi-facetted that your code really has to take a ton of scenarios into account. I'll try and make line-notes below so you can see...
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-103317824
Introduced the "fetch all" mechanism as described on the link you found.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-104870675
I used checkstyle static code analyzer to identify number of style issues (including the ones you raised) and fixed them.
However, throwing out DefaultHttpClient didn't work for some weird reason. Strangely, my Eclipse does not even show it is deprecated. :(
The following piece of code gives compiler error "cannot find symbol" though correct jar is in classpath etc.
HttpClientBuilder httpclientBuilder = HttpClientBuilder().create();
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-102836764
If you run a broad query like "SELECT * FROM [table]" then it should be able to get the results and not apply a limit. Not necesarily as part of the same request (that would in deed cause huge memory issues) but then using a paging or cursor approach. I did a quick google on "solr paging" and this seems quite doable: https://cwiki.apache.org/confluence/display/solr/Pagination+of+Results
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569521
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
+ String expr = filterItem.getExpression();
+
+ if (expr == null){
+ expr = filterItem.toString();
+ expr = expr.replaceAll("[a-z_0-9]+\\.","");
+ }
+
+ expr = expr.replace(" or ", " OR ").replace(" and ", " AND ").replaceAll("=",":");
+
+ String joiner = (isFirst == true) ? " " : " AND ";
+ queryStr += joiner + expr;
+
+ isFirst = false;
+ }
+ }
+
+ if (queryStr.isEmpty()) {
+ queryStr = "*:*";
+ }
+
+ Integer limitObject = q.getMaxRows();
+ int limit = DEFAULT_LIMIT;
--- End diff --
Setting a default limit means that this module behaves differently than other modules. Why is this necesary? It kinda breaks the polymorphic nature of all DataContext implementations - we'd want to facilitate a _write once, run on any datastore_ kind of API.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-98772705
Thank you for the detailed comments. I concur in part - some corner cases may need handling. Some kinds of queries, such as joins are rarely used in Solr and the support is inefficient.
if executeQuery() is not implemented, then is there any other method which makes sense to implement? If not, do you recommend deferring the release of this till all corner cases are handled?
The module is almost useless if the actual search does not take place in the engine itself, as users would generally use search engines for large data sets, which practically cannot be read into the MetaModel application space and searched in the MM layer. Hence, I was wondering if it has to be a all or nothing approach and if we could throw exceptions for unsupported query patterns and incrementally handle the rare query patterns too.
Do let me know your thoughts.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29596247
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
--- End diff --
Tested this with precedence cases (bracketed) and it works.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-104020512
Sorry that I keep asking you for changes like this, we'll be happy at the end of it all :-)
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569699
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
+ String expr = filterItem.getExpression();
+
+ if (expr == null){
+ expr = filterItem.toString();
+ expr = expr.replaceAll("[a-z_0-9]+\\.","");
+ }
+
+ expr = expr.replace(" or ", " OR ").replace(" and ", " AND ").replaceAll("=",":");
+
+ String joiner = (isFirst == true) ? " " : " AND ";
+ queryStr += joiner + expr;
+
+ isFirst = false;
+ }
+ }
+
+ if (queryStr.isEmpty()) {
+ queryStr = "*:*";
+ }
+
+ Integer limitObject = q.getMaxRows();
+ int limit = DEFAULT_LIMIT;
+
+ if (limitObject != null) {
+ limit = (int) limitObject;
+ }
+
+ final List<GroupByItem> groupByItems = q.getGroupByClause().getItems();
+
+ if (groupByItems.size() > 0) {
+ query.clear();
+ query.setQuery(queryStr);
+ query.setFacet(true);
+
+ for (int i = 0; i < groupByItems.size(); i++) {
+ String facetField = groupByItems.get(i).getSelectItem()
+ .getColumn().getName();
+ query.addFacetField(facetField);
+ }
+
+ setOrder(q, query, QUERYTYPE.FACET);
+
+ query.setFacetLimit(limit);
+
+ try {
+ HttpSolrServer server = initSolrServer();
+ response = server.query(query);
--- End diff --
At this point there has been no regard to the HAVING clause of the query. There might be HAVING items.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ardlema <gi...@git.apache.org>.
Github user ardlema commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-169693523
@kaspersorensen @ashish-m-yh this PR seems to be here for a long time.
Shall we close it?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-104020116
A couple of remarks on code style:
* In eclipse I see a bunch of warnings that should be fixed:
* Unused variables (multiple)
* Unused imports (multiple)
* Deprecated classes (DefaultHttpClient)
* Use of raw type generics (multiple)
* Resource leak: HttpClient is never used
* Unstandard naming: QUERYTYPE is entirely uppercase, should be QueryType
* For me to read a very inconsistent use of "this" keyword and instance variables - difficult state management especially in the dataset class.
Bringing these items to order is for me a prerequisite for doing a proper review because readability isn't very good as long as I have to deal with so many warnings etc.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569636
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
+ String expr = filterItem.getExpression();
+
+ if (expr == null){
+ expr = filterItem.toString();
+ expr = expr.replaceAll("[a-z_0-9]+\\.","");
+ }
+
+ expr = expr.replace(" or ", " OR ").replace(" and ", " AND ").replaceAll("=",":");
+
+ String joiner = (isFirst == true) ? " " : " AND ";
+ queryStr += joiner + expr;
+
+ isFirst = false;
+ }
+ }
+
+ if (queryStr.isEmpty()) {
+ queryStr = "*:*";
+ }
+
+ Integer limitObject = q.getMaxRows();
+ int limit = DEFAULT_LIMIT;
+
+ if (limitObject != null) {
+ limit = (int) limitObject;
+ }
+
+ final List<GroupByItem> groupByItems = q.getGroupByClause().getItems();
+
+ if (groupByItems.size() > 0) {
+ query.clear();
+ query.setQuery(queryStr);
+ query.setFacet(true);
+
+ for (int i = 0; i < groupByItems.size(); i++) {
+ String facetField = groupByItems.get(i).getSelectItem()
+ .getColumn().getName();
--- End diff --
You're guaranteed that there will be a SelectItem, but not that it has a column reference. Acknowledged, it's not very common to have GROUP BY without a column reference, but it _could_ be a corner case of some sort and then you'll run into a NPE here.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569465
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
+ String expr = filterItem.getExpression();
+
+ if (expr == null){
+ expr = filterItem.toString();
--- End diff --
This is the most normal case, yet you are not checking on what kind of operator is being applied in the WHERE item. Does Solr support the same operators as MetaModel? To rewrite it into Solr syntax I would imagine a switch-case here on the basis of the operator type.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29596296
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
+ String expr = filterItem.getExpression();
+
+ if (expr == null){
+ expr = filterItem.toString();
+ expr = expr.replaceAll("[a-z_0-9]+\\.","");
+ }
+
+ expr = expr.replace(" or ", " OR ").replace(" and ", " AND ").replaceAll("=",":");
+
+ String joiner = (isFirst == true) ? " " : " AND ";
+ queryStr += joiner + expr;
+
+ isFirst = false;
+ }
+ }
+
+ if (queryStr.isEmpty()) {
+ queryStr = "*:*";
+ }
+
+ Integer limitObject = q.getMaxRows();
+ int limit = DEFAULT_LIMIT;
+
+ if (limitObject != null) {
+ limit = (int) limitObject;
+ }
+
+ final List<GroupByItem> groupByItems = q.getGroupByClause().getItems();
+
+ if (groupByItems.size() > 0) {
+ query.clear();
+ query.setQuery(queryStr);
+ query.setFacet(true);
+
+ for (int i = 0; i < groupByItems.size(); i++) {
+ String facetField = groupByItems.get(i).getSelectItem()
+ .getColumn().getName();
--- End diff --
Fair point
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29112578
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for ElasticSearch analytics engine.
+ *
+ * ElasticSearch has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ public static final String FIELD_ID = "id";
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum queryTypes {
--- End diff --
Normally any type (also enum) is written in uppercase and in singular grammatical form.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569934
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
+ String expr = filterItem.getExpression();
+
+ if (expr == null){
+ expr = filterItem.toString();
+ expr = expr.replaceAll("[a-z_0-9]+\\.","");
+ }
+
+ expr = expr.replace(" or ", " OR ").replace(" and ", " AND ").replaceAll("=",":");
+
+ String joiner = (isFirst == true) ? " " : " AND ";
+ queryStr += joiner + expr;
+
+ isFirst = false;
+ }
+ }
+
+ if (queryStr.isEmpty()) {
+ queryStr = "*:*";
+ }
+
+ Integer limitObject = q.getMaxRows();
+ int limit = DEFAULT_LIMIT;
+
+ if (limitObject != null) {
+ limit = (int) limitObject;
+ }
+
+ final List<GroupByItem> groupByItems = q.getGroupByClause().getItems();
+
+ if (groupByItems.size() > 0) {
--- End diff --
Group by is one way of determining the faceting, but there is also another - the DISTINCT operator on the SELECT clause. If you have a DISTINCT situation, then I guess you might also need to apply faceting (or at least safeguard this code to not run without regard of the DISTINCT situation).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29271324
--- Diff: solr/src/test/java/org/apache/metamodel/solr/SolrDataContextTest1.java ---
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.swing.table.TableModel;
+
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetTableModel;
+import org.apache.metamodel.data.FilteredDataSet;
+import org.apache.metamodel.data.InMemoryDataSet;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.Table;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class SolrDataContextTest1 {
--- End diff --
Have made necessary changes and committed the code with correct filename and indentation.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29240261
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for ElasticSearch analytics engine.
+ *
+ * ElasticSearch has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ public static final String FIELD_ID = "id";
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum queryTypes {
--- End diff --
Bit of Java rustiness there. Fixed it.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29240046
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for ElasticSearch analytics engine.
--- End diff --
My bad! Fixed this too
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-98796448
Hi Ashish,
I know that doing a JOIN is pretty unusual in "normal" mode of Solr. But in my opinion we are also not catering to the normal cases. MetaModel exposes an API that is useful especially because it is the _same_ API across all datastores. That means our users often use MetaModel in scenarios where they might just as well use a relational database or an Excel sheet as the backing store. The benefit they get with MetaModel is that their code to process data can run on any given datastore. That means the type of use is often pretty untypical. So having support for JOIN in Solr is actually pretty important in my opinion. I would even go as far as to say that effeciency in execution is secondary to compatibility with the API. Reason being that if people are trying to get the most performance out of their Solr system then they will anyways be going directly to Solr's primary API.
That said I do believe that there are scenarios you absolutely should support.
My suggestion is just that you start small and get something that functionally fulfills the whole API. Then expand on a case-by-case basis to add optimized support for various classes of queries. For instance you can start with just materializeMainSchemaTable - make it work without any default limits or anything like that. And make sure that it is cursor based so that you don't load a million records into memory but rather maintain a cursor in the dataset implementation.
Once that is working I would expand the scope to also have support for COUNT queries. And then maybe queries with just a single WHERE item. And so on and so on ... Makes sense?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29240604
--- Diff: solr/src/test/java/org/apache/metamodel/solr/SolrDataContextTest1.java ---
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.swing.table.TableModel;
+
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetTableModel;
+import org.apache.metamodel.data.FilteredDataSet;
+import org.apache.metamodel.data.InMemoryDataSet;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.Table;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class SolrDataContextTest1 {
+ private static final String url = "http://localhost:8983/solr/collection1";
--- End diff --
Fixed
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-102798147
I understand the philosophy. We may sacrifice performance to conform to a general standard. I have implemented only the necessary methods.
Note: Only necessity is to have a DEFAULT_LIMIT constant in the Solr connector, as this is a pre-requisite of Solr and I think it's reasonable to impose a limit of 10000. If we keep INTEGER.MAX_VALUE as limit, then Solr will exhaust Heap Memory on most occasions.
Please have a look and proceed accordingly.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-104367630
It's fine. Things should be proper. On further review, I found some of the issues you reported, but it's weird why my Eclipse doesn't show it when I import the mvn project. Is there some Eclipse option/configuration which I should turn on? My projects which are created purely in Eclipse do highlight these.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29239757
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataSet.java ---
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.List;
+import java.util.Set;
+import java.util.IdentityHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.metamodel.data.AbstractDataSet;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.DefaultRow;
+import org.apache.metamodel.schema.Column;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.client.solrj.response.FacetField.Count;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+
+/**
+ * {@link DataSet} implementation for ElasticSearch
+ */
+final class SolrDataSet extends AbstractDataSet {
+
+ private static final Logger logger = LoggerFactory
--- End diff --
Pushed after fixing
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29112631
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for ElasticSearch analytics engine.
+ *
+ * ElasticSearch has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ public static final String FIELD_ID = "id";
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum queryTypes {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ final Column idColumn = table.getColumnByName(FIELD_ID);
+ if (idColumn != null && idColumn instanceof MutableColumn) {
+ final MutableColumn mutableColumn = (MutableColumn) idColumn;
+ mutableColumn.setPrimaryKey(true);
+ }
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
--- End diff --
Overriding the executeQuery(...) method is a source of many painful things because the query object can represent _any_ query. That means you need to be very very careful with safe-guarding your code against unexpected scenarios. To give an example of just one thing that will go very wrong here: What if the query has multiple FROM items - or even a JOIN in it? Or what if there are aliases shared between the clauses. As a subclass of QueryPostprocessDataContext we try to liberate you from the responsibility of having to consider all such scenarios. There are a bunch of other methods available that are a lot easier to implement. And if you really want to override executeQuery(...) then add a lot of safe-guarding so that you ensure you only handle specifically the type of query that you have in mind.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29112462
--- Diff: solr/src/test/java/org/apache/metamodel/solr/SolrDataContextTest1.java ---
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.swing.table.TableModel;
+
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetTableModel;
+import org.apache.metamodel.data.FilteredDataSet;
+import org.apache.metamodel.data.InMemoryDataSet;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.Table;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class SolrDataContextTest1 {
+ private static final String url = "http://localhost:8983/solr/collection1";
--- End diff --
Indentation seems to still be tabs instead of 4 spaces
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-105260918
METAMODEL-129: Commited
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569413
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
--- End diff --
Some filter items may have children, if they represent (x OR y) kind of scenarios.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29239910
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for ElasticSearch analytics engine.
+ *
+ * ElasticSearch has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ public static final String FIELD_ID = "id";
--- End diff --
Well, this may be redundant for now, so I removed it.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29597418
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
+ String expr = filterItem.getExpression();
+
+ if (expr == null){
+ expr = filterItem.toString();
+ expr = expr.replaceAll("[a-z_0-9]+\\.","");
+ }
+
+ expr = expr.replace(" or ", " OR ").replace(" and ", " AND ").replaceAll("=",":");
+
+ String joiner = (isFirst == true) ? " " : " AND ";
+ queryStr += joiner + expr;
+
+ isFirst = false;
+ }
+ }
+
+ if (queryStr.isEmpty()) {
+ queryStr = "*:*";
+ }
+
+ Integer limitObject = q.getMaxRows();
+ int limit = DEFAULT_LIMIT;
--- End diff --
Solr has a default facet limit anyway beyond which it will not fetch facets. Hence, DEFAULT_LIMIT has to be set to force it to fetch all.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-111191915
Did you have a chance to look at this sometime?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569356
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
--- End diff --
This loop seems to check for any SelectItem that has a function and no column. But you update a variable called "selectAll". If your intent is to find out if this is a SELECT COUNT(*) FROM [table] kind of query, then there is a much safer way to implement it - to overrid the executeCountQuery(...) method of the superclass.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29112568
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,403 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for ElasticSearch analytics engine.
+ *
+ * ElasticSearch has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ public static final String FIELD_ID = "id";
--- End diff --
I don't understand why we have a special logic for adding an ID field when there is no special logic for querying it. I think it might very well even be a source of bugs in the current code because if you add a column called "id" to every table and then don't do anything to ensure that there are values in the rows you return for this "id" column, then the rows' data and metadata will be inconsistent.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569265
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
--- End diff --
The table is referred to in the Query (somewhere in the FROM clause).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:
https://github.com/apache/metamodel/pull/19
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on the pull request:
https://github.com/apache/metamodel/pull/19#issuecomment-98612112
Sorry that was a lot of nitty gritty details. I am just trying to show that implementing executeQuery(...) is very complex because the Query object is complex. I really think that you/we should start with just "the simplest thing that works" and then work our way from there. First make it work, then make it fast.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by kaspersorensen <gi...@git.apache.org>.
Github user kaspersorensen commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29569880
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
+ String expr = filterItem.getExpression();
+
+ if (expr == null){
+ expr = filterItem.toString();
+ expr = expr.replaceAll("[a-z_0-9]+\\.","");
+ }
+
+ expr = expr.replace(" or ", " OR ").replace(" and ", " AND ").replaceAll("=",":");
+
+ String joiner = (isFirst == true) ? " " : " AND ";
+ queryStr += joiner + expr;
+
+ isFirst = false;
+ }
+ }
+
+ if (queryStr.isEmpty()) {
+ queryStr = "*:*";
+ }
+
+ Integer limitObject = q.getMaxRows();
--- End diff --
Max rows is used for limit, but what about "firstRow"? There might also be an offset.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29597555
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
--- End diff --
Have implemented it
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] metamodel pull request: Fresh PR for Sor
Posted by ashish-m-yh <gi...@git.apache.org>.
Github user ashish-m-yh commented on a diff in the pull request:
https://github.com/apache/metamodel/pull/19#discussion_r29597620
--- Diff: solr/src/main/java/org/apache/metamodel/solr/SolrDataContext.java ---
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.solr;
+
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.metamodel.DataContext;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.query.Query;
+import org.apache.metamodel.QueryPostprocessDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.DataSetHeader;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterClause;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.GroupByItem;
+import org.apache.metamodel.query.OrderByItem;
+import org.apache.metamodel.query.SelectItem;
+import org.apache.metamodel.query.LogicalOperator;
+import org.apache.metamodel.query.OperatorType;
+import org.apache.metamodel.query.FunctionType;
+import org.apache.metamodel.schema.Column;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.MutableColumn;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.MutableTable;
+import org.apache.metamodel.schema.Schema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.common.SolrDocumentList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/**
+ * DataContext implementation for Solr analytics engine.
+ *
+ * Solr has a data storage structure hierarchy that briefly goes like
+ * this:
+ * <ul>
+ * <li>Index</li>
+ * <li>Document type (short: Type) (within an index)</li>
+ * <li>Documents (of a particular type)</li>
+ * </ul>
+ *
+ * When instantiating this DataContext, an index name is provided. Within this
+ * index, each document type is represented as a table.
+ *
+ * This implementation supports either automatic discovery of a schema or manual
+ * specification of a schema, through the {@link SimpleTableDef} class.
+ */
+
+public class SolrDataContext extends QueryPostprocessDataContext implements
+ DataContext {
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(SolrDataContext.class);
+
+ private final SimpleTableDef tableDef;
+ private final String indexName;
+ private final String url;
+
+ private SolrQuery query = new SolrQuery();
+
+ private static enum QUERYTYPE {
+ ROW, FACET
+ };
+
+ private static final int MAX_RETRIES = 0;
+ private static final int SOCK_TIMEOUT = 1000;
+ private static final int CONN_TIMEOUT = 5000;
+ private static final int DEFAULT_LIMIT = 1000;
+
+ public SolrDataContext(String url, String indexName) {
+ this.url = url;
+ this.indexName = indexName;
+ this.tableDef = detectSchema(url, indexName);
+ }
+
+ private SimpleTableDef detectSchema(String url, String indexName) {
+ SimpleTableDef tableDef = null;
+
+ url += "/schema";
+
+ try {
+ HttpClient httpclient = new DefaultHttpClient();
+ HttpGet request = new HttpGet(url);
+ HttpResponse response = httpclient.execute(request);
+
+ InputStreamReader inputStream = new InputStreamReader(response
+ .getEntity().getContent());
+
+ ObjectReader reader = new ObjectMapper().reader(HashMap.class);
+ Map<String, Object> outerMap = reader.readValue(inputStream);
+
+ Map<String, Object> schemaMap = null;
+
+ if (outerMap != null) {
+ schemaMap = (Map) outerMap.get("schema");
+ }
+
+ if (schemaMap != null) {
+ List<Map<String, String>> fieldMapList = (List) schemaMap
+ .get("fields");
+ List<String> columnNames = new ArrayList<String>();
+ List<ColumnType> columnTypes = new ArrayList<ColumnType>();
+
+ for (int i = 0; i < fieldMapList.size(); i++) {
+ Map<String, String> fieldMap = fieldMapList.get(i);
+
+ columnNames.add(fieldMap.get("name"));
+ columnTypes.add(ColumnType.STRING);
+ }
+
+ String[] columnNamesArr = new String[columnNames.size()];
+ columnNamesArr = columnNames.toArray(columnNamesArr);
+
+ ColumnType[] columnTypesArr = new ColumnType[columnTypes.size()];
+ columnTypesArr = columnTypes.toArray(columnTypesArr);
+
+ tableDef = new SimpleTableDef(indexName, columnNamesArr,
+ columnTypesArr);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to parse schema", "", e);
+ }
+
+ return tableDef;
+ }
+
+ @Override
+ protected Schema getMainSchema() throws MetaModelException {
+ try {
+ final MutableSchema theSchema = new MutableSchema(
+ getMainSchemaName());
+
+ final MutableTable table = tableDef.toTable().setSchema(theSchema);
+ theSchema.addTable(table);
+
+ return theSchema;
+ } catch (Exception e) {
+ throw new MetaModelException("Schema retrieval failed " + e);
+ }
+ }
+
+ @Override
+ protected Number executeCountQuery(Table table,
+ List<FilterItem> whereItems, boolean functionApproximationAllowed) {
+ String queryStr = "*:*";
+
+ if (whereItems != null) {
+ for (int i = 0; i < whereItems.size(); i++) {
+ FilterItem whereItem = whereItems.get(i);
+ queryStr = whereItem.toString();
+ }
+ }
+
+ QueryResponse response = selectRows(table, null, queryStr, 0, 0);
+ return response.getResults().getNumFound();
+ }
+
+ @Override
+ public DataSet executeQuery(Query q) throws MetaModelException {
+ String queryStr = "";
+ QueryResponse response = null;
+
+ Schema schema = getMainSchema();
+ Table table = schema.getTable(0);
+ Column[] columns = table.getColumns();
+
+ boolean selectAll = true;
+ List<SelectItem> selectItems = q.getSelectClause().getItems();
+
+ for (SelectItem selectItem : selectItems) {
+ if (selectItem.getFunction() != null
+ || selectItem.getColumn() == null) {
+ selectAll = false;
+ break;
+ }
+ }
+
+ if (selectItems.size() == 1 && selectAll == false) {
+ return super.executeQuery(q);
+ }
+
+ FilterClause clause = q.getWhereClause();
+
+ if (clause != null) {
+ List<FilterItem> filterItemList = clause.getItems();
+
+ boolean isFirst = true;
+
+ for (FilterItem filterItem : filterItemList) {
+ String expr = filterItem.getExpression();
+
+ if (expr == null){
+ expr = filterItem.toString();
--- End diff --
Tested with the known cases
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---