You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2014/11/13 12:49:28 UTC
[01/50] incubator-lens git commit: Increase size of userquery field
in finished_queries
Repository: incubator-lens
Updated Branches:
refs/heads/toapache [created] d34a8f7b9
Increase size of userquery field in finished_queries
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/65d9036f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/65d9036f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/65d9036f
Branch: refs/heads/toapache
Commit: 65d9036f99956b0a9396fe4887106ea9d9f2f483
Parents: 10f6853
Author: jdhok <ja...@inmobi.com>
Authored: Wed Nov 5 15:27:45 2014 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Wed Nov 5 15:27:45 2014 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/server/query/LensServerDAO.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/65d9036f/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index d7c914c..4dc8daf 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -96,7 +96,7 @@ public class LensServerDAO {
*/
public void createFinishedQueriesTable() throws Exception {
String sql = "CREATE TABLE if not exists finished_queries (handle varchar(255) not null unique,"
- + "userquery varchar(255) not null," + "submitter varchar(255) not null," + "starttime bigint, "
+ + "userquery varchar(10000) not null," + "submitter varchar(255) not null," + "starttime bigint, "
+ "endtime bigint," + "result varchar(255)," + "status varchar(255), " + "metadata varchar(100000), "
+ "rows int, " + "errormessage varchar(10000), " + "driverstarttime bigint, " + "driverendtime bigint, "
+ "metadataclass varchar(10000)," + "queryname varchar(255)," + "submissiontime bigint" + ")";
[25/50] incubator-lens git commit: Merge pull request #173 from
InMobi/svc_stop_npe
Posted by am...@apache.org.
Merge pull request #173 from InMobi/svc_stop_npe
Fixed errors when shutting down server in case of failed init
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/e21f7501
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/e21f7501
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/e21f7501
Branch: refs/heads/toapache
Commit: e21f7501b768c6816aeb35da93c638de93795841
Parents: fab4c56 b690a92
Author: Amareshwari <am...@inmobi.com>
Authored: Tue Nov 11 13:29:47 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Tue Nov 11 13:29:47 2014 +0530
----------------------------------------------------------------------
.../apache/lens/server/EventServiceImpl.java | 20 +++----
.../org/apache/lens/server/LensServices.java | 34 ++++++------
.../apache/lens/server/MetricsServiceImpl.java | 17 ++++--
.../lens/server/TestStartupOnMetastoreDown.java | 56 ++++++++++++++++++++
4 files changed, 98 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
[15/50] incubator-lens git commit: Added verification for repeat runs
of same task
Posted by am...@apache.org.
Added verification for repeat runs of same task
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/acb26187
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/acb26187
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/acb26187
Branch: refs/heads/toapache
Commit: acb26187ac98004069b3aa16b625417086d16bdc
Parents: 31d41d9
Author: jdhok <ja...@inmobi.com>
Authored: Fri Nov 7 14:40:52 2014 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Fri Nov 7 14:40:52 2014 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/ml/task/MLTask.java | 9 +++-
.../java/org/apache/lens/ml/TestMLResource.java | 45 +++++++++++++++++++-
2 files changed, 52 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/acb26187/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
index 85fd01d..d2bed0a 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
@@ -127,6 +127,12 @@ public class MLTask implements Runnable {
@Getter
public String password;
+ @Getter
+ private String modelID;
+
+ @Getter
+ private String reportID;
+
/**
* Use ExampleTask.Builder to create an instance
*/
@@ -254,11 +260,12 @@ public class MLTask implements Runnable {
String trainerArgs[] = buildTrainingArgs();
LOG.info("Starting task " + taskID + " trainer args: " + Arrays.toString(trainerArgs));
- String modelID = ml.train(trainingTable, algorithm, trainerArgs);
+ modelID = ml.train(trainingTable, algorithm, trainerArgs);
printModelMetadata(taskID, modelID);
LOG.info("Starting test " + taskID);
MLTestReport testReport = ml.testModel(sessionHandle, trainingTable, algorithm, modelID, outputTable);
+ reportID = testReport.getReportID();
printTestReport(taskID, testReport);
saveTask();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/acb26187/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
index 8ebff9c..a0abe93 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
@@ -20,9 +20,12 @@ package org.apache.lens.ml;
import java.io.File;
import java.net.URI;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Application;
@@ -30,7 +33,10 @@ import javax.ws.rs.core.Application;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hive.service.Service;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.client.LensConnectionParams;
@@ -54,7 +60,6 @@ import org.apache.lens.server.session.HiveSessionService;
import org.apache.lens.server.session.SessionResource;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
-import org.glassfish.jersey.server.ResourceConfig;
import org.testng.Assert;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeMethod;
@@ -202,6 +207,44 @@ public class TestMLResource extends LensJerseyTest {
task.run();
Assert.assertEquals(task.getTaskState(), MLTask.State.SUCCESSFUL);
+ String firstModelID = task.getModelID();
+ String firstReportID = task.getReportID();
+ Assert.assertNotNull(firstReportID);
+ Assert.assertNotNull(firstModelID);
+
+ taskBuilder = new MLTask.Builder();
+ taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn).outputTable(outputTable)
+ .serverLocation(getBaseUri().toString()).sessionHandle(mlClient.getSessionHandle()).trainingTable(tableName)
+ .userName("foo@localhost").password("bar");
+ taskBuilder.addFeatureColumn("feature_1").addFeatureColumn("feature_2").addFeatureColumn("feature_3");
+
+ MLTask anotherTask = taskBuilder.build();
+
+ LOG.info("Created second task " + anotherTask.toString());
+ anotherTask.run();
+
+ String secondModelID = anotherTask.getModelID();
+ String secondReportID = anotherTask.getReportID();
+ Assert.assertNotNull(secondModelID);
+ Assert.assertNotNull(secondReportID);
+
+ Hive metastoreClient = Hive.get(conf);
+ Table outputHiveTable = metastoreClient.getTable(outputTable);
+ List<Partition> partitions = metastoreClient.getPartitions(outputHiveTable);
+
+ Assert.assertNotNull(partitions);
+
+ int i = 0;
+ Set<String> partReports = new HashSet<String>();
+ for (Partition part : partitions) {
+ LOG.info("@@PART#" + i + " " + part.getSpec().toString());
+ partReports.add(part.getSpec().get("part_testid"));
+ }
+
+ // Verify partitions created for each run
+ Assert.assertTrue(partReports.contains(firstReportID), firstReportID + " first partition not there");
+ Assert.assertTrue(partReports.contains(secondReportID), secondReportID + " second partition not there");
+
LOG.info("Completed task run");
}
[08/50] incubator-lens git commit: Fix test failure in
TestQueryService
Posted by am...@apache.org.
Fix test failure in TestQueryService
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/01a5c35f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/01a5c35f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/01a5c35f
Branch: refs/heads/toapache
Commit: 01a5c35f2d7919b0445bbb38097b159077f6ab43
Parents: 2762a5b
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Fri Nov 7 11:03:43 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Fri Nov 7 11:05:18 2014 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/server/query/TestQueryService.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/01a5c35f/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 3c4f437..89321e8 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -417,7 +417,7 @@ public class TestQueryService extends LensJerseyTest {
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), QueryPlan.class);
Assert.assertEquals(plan.getNumSels(), 1);
Assert.assertEquals(plan.getTablesQueried().size(), 1);
- Assert.assertTrue(plan.getTablesQueried().get(0).equalsIgnoreCase(testTable));
+ Assert.assertTrue(plan.getTablesQueried().get(0).endsWith(testTable.toLowerCase()));
Assert.assertNull(plan.getPrepareHandle());
// Test explain and prepare
@@ -435,7 +435,7 @@ public class TestQueryService extends LensJerseyTest {
QueryPlan.class);
Assert.assertEquals(plan2.getNumSels(), 1);
Assert.assertEquals(plan2.getTablesQueried().size(), 1);
- Assert.assertTrue(plan2.getTablesQueried().get(0).equalsIgnoreCase(testTable));
+ Assert.assertTrue(plan2.getTablesQueried().get(0).endsWith(testTable.toLowerCase()));
Assert.assertNotNull(plan2.getPrepareHandle());
}
@@ -611,7 +611,7 @@ public class TestQueryService extends LensJerseyTest {
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), QueryPlan.class);
Assert.assertEquals(plan.getNumSels(), 1);
Assert.assertEquals(plan.getTablesQueried().size(), 1);
- Assert.assertTrue(plan.getTablesQueried().get(0).equalsIgnoreCase(testTable));
+ Assert.assertTrue(plan.getTablesQueried().get(0).endsWith(testTable.toLowerCase()));
Assert.assertNotNull(plan.getPrepareHandle());
LensPreparedQuery ctx = target.path(plan.getPrepareHandle().toString()).queryParam("sessionid", lensSessionId)
[33/50] incubator-lens git commit: Moving Mock classes to a more
convenient/accessible location: inside lens-server-api
Posted by am...@apache.org.
Moving Mock classes to a more convenient/accessible location: inside lens-server-api
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/180f6b41
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/180f6b41
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/180f6b41
Branch: refs/heads/toapache
Commit: 180f6b4146b2dd17cae02df3ae434dcd52154d90
Parents: 90dfa2a
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Wed Nov 12 16:02:55 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Wed Nov 12 16:02:55 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/cube/MockDriver.java | 345 ------------------
.../lens/driver/cube/TestMinCostSelector.java | 33 +-
.../apache/lens/driver/cube/TestRewriting.java | 3 +-
lens-cube/src/test/resources/hive-site.xml | 2 +-
.../server/api/driver/MinQueryCostSelector.java | 53 +--
.../lens/server/api/driver/MockDriver.java | 361 +++++++++++++++++++
.../lens/server/api/driver/MockFailDriver.java | 37 ++
.../server/api/driver/MockQueryContext.java | 30 ++
8 files changed, 453 insertions(+), 411 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/180f6b41/lens-cube/src/test/java/org/apache/lens/driver/cube/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/MockDriver.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/MockDriver.java
deleted file mode 100644
index 5b470de..0000000
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/MockDriver.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.driver.cube;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hive.service.cli.ColumnDescriptor;
-import org.apache.lens.api.LensException;
-import org.apache.lens.api.query.QueryCost;
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryPrepareHandle;
-import org.apache.lens.api.query.ResultRow;
-import org.apache.lens.server.api.driver.*;
-import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
-import org.apache.lens.server.api.events.LensEventListener;
-import org.apache.lens.server.api.query.PreparedQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
-
-/**
- * The Class MockDriver.
- */
-public class MockDriver implements LensDriver {
-
- /** The conf. */
- Configuration conf;
-
- /** The query. */
- String query;
-
- /** The io test val. */
- private int ioTestVal = -1;
-
- /**
- * Instantiates a new mock driver.
- */
- public MockDriver() {
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#configure(org.apache.hadoop.conf.Configuration)
- */
- @Override
- public void configure(Configuration conf) throws LensException {
- this.conf = conf;
- ioTestVal = conf.getInt("mock.driver.test.val", -1);
- }
-
- /**
- * The Class MockQueryPlan.
- */
- static class MockQueryPlan extends DriverQueryPlan {
-
- /** The query. */
- String query;
-
- /**
- * Instantiates a new mock query plan.
- *
- * @param query
- * the query
- */
- MockQueryPlan(String query) {
- this.query = query;
- setPrepareHandle(new QueryPrepareHandle(UUID.randomUUID()));
- }
-
- @Override
- public String getPlan() {
- return query;
- }
-
- @Override
- public QueryCost getCost() {
- return new QueryCost(0L, 0.0);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#explain(java.lang.String, org.apache.hadoop.conf.Configuration)
- */
- @Override
- public DriverQueryPlan explain(String query, Configuration conf) throws LensException {
- return new MockQueryPlan(query);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#updateStatus(org.apache.lens.server.api.query.QueryContext)
- */
- @Override
- public void updateStatus(QueryContext context) throws LensException {
- context.getDriverStatus().setProgress(1.0);
- context.getDriverStatus().setStatusMessage("Done");
- context.getDriverStatus().setState(DriverQueryState.SUCCESSFUL);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#cancelQuery(org.apache.lens.api.query.QueryHandle)
- */
- @Override
- public boolean cancelQuery(QueryHandle handle) throws LensException {
- return false;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#closeQuery(org.apache.lens.api.query.QueryHandle)
- */
- @Override
- public void closeQuery(QueryHandle handle) throws LensException {
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#close()
- */
- @Override
- public void close() throws LensException {
- }
-
- /**
- * Add a listener for driver events.
- *
- * @param driverEventListener
- * the driver event listener
- */
- @Override
- public void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener) {
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#prepare(org.apache.lens.server.api.query.PreparedQueryContext)
- */
- @Override
- public void prepare(PreparedQueryContext pContext) throws LensException {
- // TODO Auto-generated method stub
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.lens.server.api.driver.LensDriver#explainAndPrepare(org.apache.lens.server.api.query.PreparedQueryContext
- * )
- */
- @Override
- public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws LensException {
- DriverQueryPlan p = new MockQueryPlan(pContext.getDriverQuery());
- p.setPrepareHandle(pContext.getPrepareHandle());
- return p;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#closePreparedQuery(org.apache.lens.api.query.QueryPrepareHandle)
- */
- @Override
- public void closePreparedQuery(QueryPrepareHandle handle) throws LensException {
- // TODO Auto-generated method stub
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#execute(org.apache.lens.server.api.query.QueryContext)
- */
- @Override
- public LensResultSet execute(QueryContext context) throws LensException {
- this.query = context.getDriverQuery();
- return new PersistentResultSet() {
-
- @Override
- public int size() throws LensException {
- // TODO Auto-generated method stub
- return 0;
- }
-
- @Override
- public LensResultSetMetadata getMetadata() throws LensException {
- // TODO Auto-generated method stub
- return new LensResultSetMetadata() {
-
- @Override
- public List<ColumnDescriptor> getColumns() {
- // TODO Auto-generated method stub
- return null;
- }
- };
- }
-
- @Override
- public String getOutputPath() throws LensException {
- // TODO Auto-generated method stub
- return null;
- }
- };
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#executeAsync(org.apache.lens.server.api.query.QueryContext)
- */
- @Override
- public void executeAsync(QueryContext context) throws LensException {
- this.query = context.getDriverQuery();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#fetchResultSet(org.apache.lens.server.api.query.QueryContext)
- */
- @Override
- public LensResultSet fetchResultSet(QueryContext context) throws LensException {
- return new InMemoryResultSet() {
-
- @Override
- public int size() throws LensException {
- // TODO Auto-generated method stub
- return 0;
- }
-
- @Override
- public LensResultSetMetadata getMetadata() throws LensException {
- return new LensResultSetMetadata() {
-
- @Override
- public List<ColumnDescriptor> getColumns() {
- // TODO Auto-generated method stub
- return null;
- }
- };
- }
-
- @Override
- public void setFetchSize(int size) throws LensException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public ResultRow next() throws LensException {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public boolean hasNext() throws LensException {
- // TODO Auto-generated method stub
- return false;
- }
- };
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#closeResultSet(org.apache.lens.api.query.QueryHandle)
- */
- @Override
- public void closeResultSet(QueryHandle handle) throws LensException {
- // TODO Auto-generated method stub
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.lens.server.api.driver.LensDriver#registerForCompletionNotification(org.apache.lens.api.query.QueryHandle
- * , long, org.apache.lens.server.api.driver.QueryCompletionListener)
- */
- @Override
- public void registerForCompletionNotification(QueryHandle handle, long timeoutMillis, QueryCompletionListener listener)
- throws LensException {
- // TODO Auto-generated method stub
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.io.Externalizable#readExternal(java.io.ObjectInput)
- */
- @Override
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- ioTestVal = in.readInt();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput)
- */
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- out.writeInt(ioTestVal);
- }
-
- public int getTestIOVal() {
- return ioTestVal;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/180f6b41/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
index 66de80c..a3df316 100644
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
+++ b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
@@ -25,9 +25,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.lens.api.LensException;
-import org.apache.lens.server.api.driver.DriverQueryPlan;
-import org.apache.lens.server.api.driver.LensDriver;
-import org.apache.lens.server.api.driver.MinQueryCostSelector;
+import org.apache.lens.server.api.driver.*;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -35,27 +33,8 @@ import org.testng.annotations.Test;
* The Class TestMinCostSelector.
*/
public class TestMinCostSelector {
-
- /**
- * The Class MockFailDriver.
- */
- static class MockFailDriver extends MockDriver {
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.driver.cube.MockDriver#explain(java.lang.String, org.apache.hadoop.conf.Configuration)
- */
- public DriverQueryPlan explain(String query, Configuration conf) throws LensException {
- throw new LensException("failing!");
- }
- }
-
- /**
- * Test min cost selector.
- */
@Test
- public void testMinCostSelector() {
+ public void testMinCostSelector() throws LensException {
MinQueryCostSelector selector = new MinQueryCostSelector();
List<LensDriver> drivers = new ArrayList<LensDriver>();
Map<LensDriver, String> driverQueries = new HashMap<LensDriver, String>();
@@ -69,21 +48,21 @@ public class TestMinCostSelector {
drivers.add(d1);
drivers.add(d2);
driverQueries.put(d1, "test query");
- LensDriver selected = selector.select(drivers, driverQueries, conf);
+ LensDriver selected = selector.select(new MockQueryContext(driverQueries), conf);
Assert.assertEquals(d1, selected);
driverQueries.put(d2, "test query");
driverQueries.remove(d1);
- selected = selector.select(drivers, driverQueries, conf);
+ selected = selector.select(new MockQueryContext(driverQueries), conf);
Assert.assertEquals(d2, selected);
drivers.add(fd1);
driverQueries.put(fd1, "test query");
- selected = selector.select(drivers, driverQueries, conf);
+ selected = selector.select(new MockQueryContext(driverQueries), conf);
Assert.assertEquals(d2, selected);
drivers.add(fd2);
driverQueries.put(fd2, "test query");
- selected = selector.select(drivers, driverQueries, conf);
+ selected = selector.select(new MockQueryContext(driverQueries), conf);
Assert.assertEquals(d2, selected);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/180f6b41/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
index 4bc2d23..538d27d 100644
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
+++ b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
@@ -31,10 +31,11 @@ import org.apache.lens.api.LensException;
import org.apache.lens.cube.parse.CubeQueryContext;
import org.apache.lens.cube.parse.CubeQueryRewriter;
import org.apache.lens.cube.parse.HQLParser;
-import org.apache.lens.driver.cube.RewriteUtil;
import org.apache.lens.server.api.driver.LensDriver;
import static org.mockito.Matchers.any;
+
+import org.apache.lens.server.api.driver.MockDriver;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/180f6b41/lens-cube/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/hive-site.xml b/lens-cube/src/test/resources/hive-site.xml
index 5624f03..b450ec2 100644
--- a/lens-cube/src/test/resources/hive-site.xml
+++ b/lens-cube/src/test/resources/hive-site.xml
@@ -25,7 +25,7 @@
<property>
<name>lens.server.drivers</name>
- <value>org.apache.lens.driver.cube.MockDriver</value>
+ <value>org.apache.lens.server.api.driver.MockDriver</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/180f6b41/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
index a280c1b..1377d81 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
@@ -20,6 +20,7 @@ package org.apache.lens.server.api.driver;
import org.apache.hadoop.conf.Configuration;
import org.apache.lens.api.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.log4j.Logger;
import java.util.Collection;
@@ -29,55 +30,33 @@ import java.util.Map;
public class MinQueryCostSelector implements DriverSelector {
public static final Logger LOG = Logger.getLogger(MinQueryCostSelector.class);
-
/**
* Returns the driver that has the minimum query cost.
*
- * @param drivers
- * the drivers
- * @param driverQueries
- * the driver queries
+ * @param ctx
+ * the context
* @param conf
* the conf
* @return the lens driver
*/
@Override
- public LensDriver select(Collection<LensDriver> drivers, final Map<LensDriver, String> driverQueries,
+ public LensDriver select(final AbstractQueryContext ctx,
final Configuration conf) {
- return Collections.min(drivers, new Comparator<LensDriver>() {
+ return Collections.min(ctx.getDriverQueries().keySet(), new Comparator<LensDriver>() {
@Override
public int compare(LensDriver d1, LensDriver d2) {
- DriverQueryPlan c1 = null;
- DriverQueryPlan c2 = null;
- // Handle cases where the queries can be null because the storages are not
- // supported.
- if (driverQueries.get(d1) == null) {
- return 1;
- }
- if (driverQueries.get(d2) == null) {
- return -1;
- }
- try {
- c1 = d1.explain(driverQueries.get(d1), conf);
- } catch (LensException e) {
- LOG.warn("Explain query:" + driverQueries.get(d1) + " on Driver:" + d1.getClass().getSimpleName()
- + " failed", e);
- }
- try {
- c2 = d2.explain(driverQueries.get(d2), conf);
- } catch (LensException e) {
- LOG.warn("Explain query:" + driverQueries.get(d2) + " on Driver:" + d2.getClass().getSimpleName()
- + " failed", e);
- }
- if (c1 == null && c2 == null) {
- return 0;
- } else if (c1 == null && c2 != null) {
- return 1;
- } else if (c1 != null && c2 == null) {
- return -1;
- }
- return c1.getCost().compareTo(c2.getCost());
+ return comparePlans(ctx.getDriverQueryPlans().get(d1), ctx.getDriverQueryPlans().get(d2));
}
});
}
+ int comparePlans(DriverQueryPlan c1, DriverQueryPlan c2) {
+ if (c1 == null && c2 == null) {
+ return 0;
+ } else if (c1 == null && c2 != null) {
+ return 1;
+ } else if (c1 != null && c2 == null) {
+ return -1;
+ }
+ return c1.getCost().compareTo(c2.getCost());
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/180f6b41/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockDriver.java
new file mode 100644
index 0000000..8487058
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -0,0 +1,361 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.driver;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.service.cli.ColumnDescriptor;
+import org.apache.lens.api.LensException;
+import org.apache.lens.api.query.QueryCost;
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryPrepareHandle;
+import org.apache.lens.api.query.ResultRow;
+import org.apache.lens.server.api.driver.*;
+import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
+import org.apache.lens.server.api.events.LensEventListener;
+import org.apache.lens.server.api.query.PreparedQueryContext;
+import org.apache.lens.server.api.query.QueryContext;
+
+/**
+ * The Class MockDriver.
+ */
+public class MockDriver implements LensDriver {
+
+ /** The conf. */
+ Configuration conf;
+
+ /** The query. */
+ String query;
+
+ /** The io test val. */
+ private int ioTestVal = -1;
+
+ /**
+ * Instantiates a new mock driver.
+ */
+ public MockDriver() {
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#configure(org.apache.hadoop.conf.Configuration)
+ */
+ @Override
+ public void configure(Configuration conf) throws LensException {
+ this.conf = conf;
+ ioTestVal = conf.getInt("mock.driver.test.val", -1);
+ }
+
+ /**
+ * The Class MockQueryPlan.
+ */
+ public static class MockQueryPlan extends DriverQueryPlan {
+
+ /** The query. */
+ String query;
+
+ /**
+ * Instantiates a new mock query plan.
+ *
+ * @param query
+ * the query
+ */
+ MockQueryPlan(String query) {
+ this.query = query;
+ setPrepareHandle(new QueryPrepareHandle(UUID.randomUUID()));
+ tableWeights.put("table1", 1.0);
+ tableWeights.put("table2", 2.0);
+ tableWeights.put("table3", 3.0);
+ }
+
+ @Getter @Setter
+ private Map<String, List<String>> partitions;
+
+// public Map<String, List<String>> getPartitions() {
+// return partitions;
+// }
+// public void setPartitions(Map<String, List<String>> partitions) {
+// this.partitions = partitions;
+// }
+
+ @Override
+ public String getPlan() {
+ return query;
+ }
+
+ @Override
+ public QueryCost getCost() {
+ return new QueryCost(0L, 0.0);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#explain(java.lang.String, org.apache.hadoop.conf.Configuration)
+ */
+ @Override
+ public DriverQueryPlan explain(String query, Configuration conf) throws LensException {
+ return new MockQueryPlan(query);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#updateStatus(org.apache.lens.server.api.query.QueryContext)
+ */
+ @Override
+ public void updateStatus(QueryContext context) throws LensException {
+ context.getDriverStatus().setProgress(1.0);
+ context.getDriverStatus().setStatusMessage("Done");
+ context.getDriverStatus().setState(DriverQueryState.SUCCESSFUL);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#cancelQuery(org.apache.lens.api.query.QueryHandle)
+ */
+ @Override
+ public boolean cancelQuery(QueryHandle handle) throws LensException {
+ return false;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#closeQuery(org.apache.lens.api.query.QueryHandle)
+ */
+ @Override
+ public void closeQuery(QueryHandle handle) throws LensException {
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#close()
+ */
+ @Override
+ public void close() throws LensException {
+ }
+
+ /**
+ * Add a listener for driver events.
+ *
+ * @param driverEventListener
+ * the driver event listener
+ */
+ @Override
+ public void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener) {
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#prepare(org.apache.lens.server.api.query.PreparedQueryContext)
+ */
+ @Override
+ public void prepare(PreparedQueryContext pContext) throws LensException {
+ // TODO Auto-generated method stub
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lens.server.api.driver.LensDriver#explainAndPrepare(org.apache.lens.server.api.query.PreparedQueryContext
+ * )
+ */
+ @Override
+ public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws LensException {
+ DriverQueryPlan p = new MockQueryPlan(pContext.getDriverQuery());
+ p.setPrepareHandle(pContext.getPrepareHandle());
+ return p;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#closePreparedQuery(org.apache.lens.api.query.QueryPrepareHandle)
+ */
+ @Override
+ public void closePreparedQuery(QueryPrepareHandle handle) throws LensException {
+ // TODO Auto-generated method stub
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#execute(org.apache.lens.server.api.query.QueryContext)
+ */
+ @Override
+ public LensResultSet execute(QueryContext context) throws LensException {
+ this.query = context.getDriverQuery();
+ return new PersistentResultSet() {
+
+ @Override
+ public int size() throws LensException {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public LensResultSetMetadata getMetadata() throws LensException {
+ // TODO Auto-generated method stub
+ return new LensResultSetMetadata() {
+
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public String getOutputPath() throws LensException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+ };
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#executeAsync(org.apache.lens.server.api.query.QueryContext)
+ */
+ @Override
+ public void executeAsync(QueryContext context) throws LensException {
+ this.query = context.getDriverQuery();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#fetchResultSet(org.apache.lens.server.api.query.QueryContext)
+ */
+ @Override
+ public LensResultSet fetchResultSet(QueryContext context) throws LensException {
+ return new InMemoryResultSet() {
+
+ @Override
+ public int size() throws LensException {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public LensResultSetMetadata getMetadata() throws LensException {
+ return new LensResultSetMetadata() {
+
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public void setFetchSize(int size) throws LensException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public ResultRow next() throws LensException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public boolean hasNext() throws LensException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+ };
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.LensDriver#closeResultSet(org.apache.lens.api.query.QueryHandle)
+ */
+ @Override
+ public void closeResultSet(QueryHandle handle) throws LensException {
+ // TODO Auto-generated method stub
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lens.server.api.driver.LensDriver#registerForCompletionNotification(org.apache.lens.api.query.QueryHandle
+ * , long, org.apache.lens.server.api.driver.QueryCompletionListener)
+ */
+ @Override
+ public void registerForCompletionNotification(QueryHandle handle, long timeoutMillis, QueryCompletionListener listener)
+ throws LensException {
+ // TODO Auto-generated method stub
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.io.Externalizable#readExternal(java.io.ObjectInput)
+ */
+ @Override
+ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ ioTestVal = in.readInt();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput)
+ */
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeInt(ioTestVal);
+ }
+
+ public int getTestIOVal() {
+ return ioTestVal;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/180f6b41/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockFailDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockFailDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockFailDriver.java
new file mode 100644
index 0000000..3346701
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockFailDriver.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.driver;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.lens.api.LensException;
+import org.apache.lens.server.api.driver.DriverQueryPlan;
+import org.apache.lens.server.api.driver.MockDriver;
+
+
+public class MockFailDriver extends MockDriver {
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.MockDriver#explain(java.lang.String, org.apache.hadoop.conf.Configuration)
+ */
+ public DriverQueryPlan explain(String query, Configuration conf) throws LensException {
+ throw new LensException("failing!");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/180f6b41/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockQueryContext.java
new file mode 100644
index 0000000..c4d6b6d
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockQueryContext.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.driver;
+
+import java.util.Map;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+
+public class MockQueryContext extends AbstractQueryContext {
+ public MockQueryContext(Map<LensDriver, String> driverQueries) throws LensException {
+ setDriverQueriesAndPlans(driverQueries);
+ }
+}
[29/50] incubator-lens git commit: addProperty calls added
Posted by am...@apache.org.
addProperty calls added
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/aea791f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/aea791f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/aea791f1
Branch: refs/heads/toapache
Commit: aea791f100fb425fbfd16bbf30325c10bf82d8f5
Parents: 77c8ca0
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Tue Nov 11 16:16:37 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Tue Nov 11 16:16:37 2014 +0530
----------------------------------------------------------------------
.../main/java/org/apache/lens/cube/metadata/CubeMeasure.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/aea791f1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java
index 5380b07..bc11cdd 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java
@@ -166,5 +166,11 @@ public abstract class CubeMeasure extends CubeColumn {
if (aggregate != null) {
props.put(MetastoreUtil.getMeasureAggrPropertyKey(getName()), aggregate);
}
+ if (min != null) {
+ props.put(MetastoreUtil.getMeasureMinPropertyKey(getName()), min.toString());
+ }
+ if (max != null) {
+ props.put(MetastoreUtil.getMeasureMaxPropertyKey(getName()), max.toString());
+ }
}
}
[34/50] incubator-lens git commit: Query Priority implemented. *
Change in DriverSelector's api to facilitate explain on all drivers is called
beforehand. * Change in QueryContext,
PreparedQueryContext and addition of DriverSelectorQueryContext, A
Posted by am...@apache.org.
Query Priority implemented.
* Change in DriverSelector's api to facilitate explain on all drivers is called beforehand.
* Change in QueryContext,PreparedQueryContext and addition of DriverSelectorQueryContext, AbstractQueryContext and ExplainQueryContext in order with the last point
* Priority Decider API and implementation for Hive Driver
* Hive Driver modifications to use Priority Decider API. Only called in async execute and surrounded with a try block.
* Accompanying test cases
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/e2d157ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/e2d157ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/e2d157ea
Branch: refs/heads/toapache
Commit: e2d157eadc1829c8e7100a09621257e3947f888a
Parents: 180f6b4
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Wed Nov 12 16:12:08 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Wed Nov 12 17:00:07 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 42 ++++++++--
.../DurationBasedQueryPriorityDecider.java | 71 ++++++++++++++++
.../apache/lens/driver/hive/TestHiveDriver.java | 36 ++++++--
.../lens/driver/hive/TestRemoteHiveDriver.java | 5 +-
.../src/test/resources/priority_tests.txt | 4 +
.../lens/server/api/driver/DriverSelector.java | 9 +-
.../api/priority/CostToPriorityRangeConf.java | 44 ++++++++++
.../api/priority/QueryPriorityDecider.java | 30 +++++++
.../lens/server/api/priority/RangeConf.java | 61 ++++++++++++++
.../server/api/query/AbstractQueryContext.java | 38 +++++++++
.../api/query/DriverSelectorQueryContext.java | 87 ++++++++++++++++++++
.../server/api/query/ExplainQueryContext.java | 28 +++++++
.../server/api/query/PreparedQueryContext.java | 25 +-----
.../lens/server/api/query/QueryContext.java | 26 +-----
.../server/query/QueryExecutionServiceImpl.java | 39 +++------
.../lens/server/query/TestQueryService.java | 16 ++--
16 files changed, 456 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 5ea20b2..d9b62f5 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -57,6 +57,8 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.events.LensEventListener;
+import org.apache.lens.driver.hive.priority.DurationBasedQueryPriorityDecider;
+import org.apache.lens.server.api.priority.QueryPriorityDecider;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.log4j.Logger;
@@ -72,13 +74,21 @@ public class HiveDriver implements LensDriver {
public static final Logger LOG = Logger.getLogger(HiveDriver.class);
/** The Constant HIVE_CONNECTION_CLASS. */
- public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
+ static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
/** The Constant HS2_CONNECTION_EXPIRY_DELAY. */
- public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hs2.connection.expiry.delay";
+ private static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hs2.connection.expiry.delay";
+
+ static final String HS2_CALCULATE_PRIORITY = "lens.driver.hs2.calculate.priority";
+ private static final String HS2_PARTITION_WEIGHT_MONTHLY = "lens.driver.hs2.priority.partition.weight.monthly";
+ private static final String HS2_PARTITION_WEIGHT_DAILY = "lens.driver.hs2.priority.partition.weight.daily";
+ private static final String HS2_PARTITION_WEIGHT_HOURLY = "lens.driver.hs2.priority.partition.weight.hourly";
// Default expiry is 10 minutes
/** The Constant DEFAULT_EXPIRY_DELAY. */
- public static final long DEFAULT_EXPIRY_DELAY = 600 * 1000;
+ private static final long DEFAULT_EXPIRY_DELAY = 600 * 1000;
+ private static final float MONTHLY_PARTITION_WEIGHT_DEFAULT = 0.5f;
+ private static final float DAILY_PARTITION_WEIGHT_DEFAULT = 0.75f;
+ private static final float HOURLY_PARTITION_WEIGHT_DEFAULT = 1.0f;
/** The driver conf. */
private HiveConf driverConf;
@@ -112,6 +122,10 @@ public class HiveDriver implements LensDriver {
/** The driver listeners. */
private List<LensEventListener<DriverEvent>> driverListeners;
+ QueryPriorityDecider queryPriorityDecider;
+
+ // package-local. Test case can change.
+ boolean whetherCalculatePriority;
/**
* The Class ConnectionExpiryRunnable.
@@ -281,6 +295,12 @@ public class HiveDriver implements LensDriver {
ThriftConnection.class);
isEmbedded = (connectionClass.getName().equals(EmbeddedThriftConnection.class.getName()));
connectionExpiryTimeout = this.driverConf.getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY);
+ whetherCalculatePriority = this.driverConf.getBoolean(HS2_CALCULATE_PRIORITY, true);
+ queryPriorityDecider = new DurationBasedQueryPriorityDecider(
+ this.driverConf.getFloat(HS2_PARTITION_WEIGHT_MONTHLY, MONTHLY_PARTITION_WEIGHT_DEFAULT),
+ this.driverConf.getFloat(HS2_PARTITION_WEIGHT_DAILY, DAILY_PARTITION_WEIGHT_DEFAULT),
+ this.driverConf.getFloat(HS2_PARTITION_WEIGHT_HOURLY, HOURLY_PARTITION_WEIGHT_DEFAULT)
+ );
}
/*
@@ -289,7 +309,7 @@ public class HiveDriver implements LensDriver {
* @see org.apache.lens.server.api.driver.LensDriver#explain(java.lang.String, org.apache.hadoop.conf.Configuration)
*/
@Override
- public DriverQueryPlan explain(final String query, final Configuration conf) throws LensException {
+ public HiveQueryPlan explain(final String query, final Configuration conf) throws LensException {
LOG.info("Explain: " + query);
Configuration explainConf = new Configuration(conf);
explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
@@ -354,12 +374,13 @@ public class HiveDriver implements LensDriver {
*
* @see org.apache.lens.server.api.driver.LensDriver#execute(org.apache.lens.server.api.query.QueryContext)
*/
+ //TODO: I'm assuming this is only called for executing explain/insert/... queries which don't ask to fetch data.
public LensResultSet execute(QueryContext ctx) throws LensException {
try {
addPersistentPath(ctx);
ctx.getConf().set("mapred.job.name", ctx.getQueryHandle().toString());
OperationHandle op = getClient().executeStatement(getSession(ctx), ctx.getDriverQuery(),
- ctx.getConf().getValByRegex(".*"));
+ ctx.getConf().getValByRegex(".*"));
LOG.info("The hive operation handle: " + op);
ctx.setDriverOpHandle(op.toString());
hiveHandles.put(ctx.getQueryHandle(), op);
@@ -395,8 +416,17 @@ public class HiveDriver implements LensDriver {
try {
addPersistentPath(ctx);
ctx.getConf().set("mapred.job.name", ctx.getQueryHandle().toString());
+ //Query is already explained.
+ if(whetherCalculatePriority) {
+ try{
+ // Inside try since non-data fetching queries can also be executed by async method.
+ ctx.getConf().set("mapred.job.priority", queryPriorityDecider.decidePriority(ctx).toString());
+ } catch(LensException e) {
+ LOG.error("could not set priority for lens session id:" + ctx.getLensSessionIdentifier(), e);
+ }
+ }
OperationHandle op = getClient().executeStatementAsync(getSession(ctx), ctx.getDriverQuery(),
- ctx.getConf().getValByRegex(".*"));
+ ctx.getConf().getValByRegex(".*"));
ctx.setDriverOpHandle(op.toString());
LOG.info("QueryHandle: " + ctx.getQueryHandle() + " HiveHandle:" + op);
hiveHandles.put(ctx.getQueryHandle(), op);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
new file mode 100644
index 0000000..5965309
--- /dev/null
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
@@ -0,0 +1,71 @@
+package org.apache.lens.driver.hive.priority;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.api.Priority;
+import org.apache.lens.server.api.priority.CostToPriorityRangeConf;
+import org.apache.lens.server.api.priority.QueryPriorityDecider;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
+ private float monthlyPartitionWeight;
+ private float dailyPartitionWeight;
+ private float hourlyPartitionWeight;
+
+ public DurationBasedQueryPriorityDecider(float mw, float dw, float hw){
+ monthlyPartitionWeight = mw;
+ dailyPartitionWeight = dw;
+ hourlyPartitionWeight = hw;
+ }
+ static final CostToPriorityRangeConf costToPriorityRangeMap =
+ // Hard Coded
+ // Arbitrary for now. Will need to tune it.
+ new CostToPriorityRangeConf("VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW");
+
+ public Priority decidePriority(AbstractQueryContext queryContext) throws LensException {
+ return costToPriorityRangeMap.get(getDurationCost(queryContext));
+ }
+
+ protected Map<String,List<String>> extractPartitions(AbstractQueryContext queryContext) throws LensException{
+ Map<String, List<String>> partitions = new HashMap<String, List<String>>();
+ for(Map.Entry<String, List<String>> entry: queryContext.getSelectedDriverQueryPlan().getPartitions().entrySet()) {
+ partitions.put(entry.getKey(), new ArrayList<String>());
+ for(String s: entry.getValue()) {
+ String[] splits = s.split("\\s+");
+ partitions.get(entry.getKey()).add(splits[splits.length - 1]); //last split.
+ }
+ }
+ return partitions;
+ }
+
+
+ float getDurationCost(AbstractQueryContext queryContext) throws LensException {
+ final Map<String, List<String>> partitions = extractPartitions(queryContext);
+ float cost = 0;
+ for(String table: partitions.keySet()) {
+ for(String partition: partitions.get(table)) {
+ if(!partition.equals("latest")) {
+ cost += queryContext.getSelectedDriverQueryPlan().getTableWeight(table) * getPartitionCost(partition);
+ }
+ }
+ }
+ return cost;
+ }
+
+ float getPartitionCost(String partition) throws LensException {
+ switch (partition.length()) {
+ case 7: //monthly
+ return 30 * monthlyPartitionWeight;
+ case 10: // daily
+ return 1 * dailyPartitionWeight;
+ case 13: // hourly
+ return (1/24) * hourlyPartitionWeight;
+ default:
+ throw new LensException("Could not recognize partition: " + partition);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 45cebde..403fbfb 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -38,18 +38,12 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.cli.ColumnDescriptor;
import org.apache.lens.api.LensException;
+import org.apache.lens.api.Priority;
import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.driver.hive.EmbeddedThriftConnection;
-import org.apache.lens.driver.hive.HiveDriver;
-import org.apache.lens.driver.hive.HiveInMemoryResultSet;
-import org.apache.lens.driver.hive.HivePersistentResultSet;
-import org.apache.lens.driver.hive.HiveQueryPlan;
-import org.apache.lens.driver.hive.ThriftConnection;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.driver.DriverQueryPlan;
-import org.apache.lens.server.api.driver.LensResultSet;
-import org.apache.lens.server.api.driver.LensResultSetMetadata;
+import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
+import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.testng.annotations.AfterTest;
@@ -115,6 +109,7 @@ public class TestHiveDriver {
conf.addResource("hivedriver-site.xml");
conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
conf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
+ conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);
driver = new HiveDriver();
driver.configure(conf);
System.out.println("TestHiveDriver created");
@@ -773,4 +768,27 @@ public class TestHiveDriver {
assertEquals(ctx.getHdfsoutPath(), path);
driver.closeQuery(plan2.getHandle());
}
+ @Test
+ public void testPriority() throws IOException, LensException {
+ final MockDriver mockDriver = new MockDriver();
+ BufferedReader br = new BufferedReader(new InputStreamReader(TestHiveDriver.class.getResourceAsStream("/priority_tests.txt")));
+ String line;
+ while((line = br.readLine()) != null) {
+ String[] kv = line.split("\\s*:\\s*");
+
+ final List<String> partitions = Arrays.asList(kv[0].trim().split("\\s*,\\s*"));
+ final Priority expected = Priority.valueOf(kv[1]);
+ AbstractQueryContext ctx = new MockQueryContext(new HashMap<LensDriver, String>(){
+ {
+ put(mockDriver, "driverQuery1");
+ }
+ });
+ ctx.setSelectedDriver(mockDriver);
+ ((MockDriver.MockQueryPlan)ctx.getDriverQueryPlans().get(mockDriver)).setPartitions(new HashMap<String, List<String>>() {
+ {
+ put("table1", partitions);
+ }});
+ Assert.assertEquals(expected, driver.queryPriorityDecider.decidePriority(ctx));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index 340e00a..c213709 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -34,9 +34,6 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.server.HiveServer2;
import org.apache.lens.api.LensException;
import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.driver.hive.HiveDriver;
-import org.apache.lens.driver.hive.RemoteThriftConnection;
-import org.apache.lens.driver.hive.ThriftConnection;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.DriverQueryPlan;
import org.apache.lens.server.api.driver.LensDriver;
@@ -132,6 +129,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
conf = new HiveConf(remoteConf);
conf.addResource("hivedriver-site.xml");
driver = new HiveDriver();
+ conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);
driver.configure(conf);
System.out.println("TestRemoteHiveDriver created");
}
@@ -234,6 +232,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
HiveConf driverConf = new HiveConf(remoteConf, TestRemoteHiveDriver.class);
driverConf.addResource("hivedriver-site.xml");
driverConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
+ driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);
final HiveDriver oldDriver = new HiveDriver();
oldDriver.configure(driverConf);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-driver-hive/src/test/resources/priority_tests.txt
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/resources/priority_tests.txt b/lens-driver-hive/src/test/resources/priority_tests.txt
new file mode 100644
index 0000000..230df95
--- /dev/null
+++ b/lens-driver-hive/src/test/resources/priority_tests.txt
@@ -0,0 +1,4 @@
+dt 2014-01-02-01: VERY_HIGH
+dt 2013-12,2014-01-01, dt 2014-01-02-00, dt 2014-01-02-01: HIGH
+dt 2013-12,2014-01, dt 2014-02, dt 2014-02-01-00: NORMAL
+dt 2013-12,2014-01, dt 2014-02, dt 2014-03, dt 2014-04, dt 2014-05: LOW
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverSelector.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverSelector.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverSelector.java
index 2d48762..3e3f61f 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverSelector.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverSelector.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.lens.server.api.query.AbstractQueryContext;
/**
* The Interface DriverSelector.
@@ -31,13 +32,11 @@ public interface DriverSelector {
/**
* Select.
*
- * @param drivers
- * the drivers
- * @param queries
- * the queries
+ * @param ctx
+ * the context
* @param conf
* the conf
* @return the lens driver
*/
- public LensDriver select(Collection<LensDriver> drivers, Map<LensDriver, String> queries, Configuration conf);
+ public LensDriver select(AbstractQueryContext ctx, Configuration conf);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
new file mode 100644
index 0000000..995666a
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.priority;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.lens.api.Priority;
+
+public class CostToPriorityRangeConf extends RangeConf<Float, Priority>{
+ public CostToPriorityRangeConf(String confValue) {
+ super(confValue);
+ }
+ @Override
+ protected Float parseKey(String s) {
+ return Float.parseFloat(s);
+ }
+
+ @Override
+ protected Priority parseValue(String s) {
+ return Priority.valueOf(s);
+ }
+
+ @Override
+ protected String getDefaultConf() {
+ return Priority.NORMAL.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
new file mode 100644
index 0000000..5fb0bda
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.priority;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.api.Priority;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.log4j.Logger;
+
+public interface QueryPriorityDecider {
+
+ public static final Logger LOG = Logger.getLogger(QueryPriorityDecider.class);
+ public Priority decidePriority(AbstractQueryContext queryContext) throws LensException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
new file mode 100644
index 0000000..c999fee
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.priority;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.lens.server.api.LensConfConstants;
+
+import java.util.TreeMap;
+
+public abstract class RangeConf<K extends Comparable<K>, V> {
+ TreeMap<K, V> map = new TreeMap<K, V>();
+ V floor;
+ RangeConf(String confValue) {
+ if(confValue == null || confValue.isEmpty()) {
+ confValue = getDefaultConf();
+ }
+ String[] split = confValue.split("\\s*,\\s*");
+ assert split.length % 2 == 1;
+ floor = parseValue(split[0]);
+ for(int i = 1; i < split.length; i += 2) {
+ map.put(parseKey(split[i]), parseValue(split[i + 1]));
+ }
+ }
+
+ protected abstract K parseKey(String s);
+ protected abstract V parseValue(String s);
+ protected abstract String getDefaultConf();
+ public V get(K key) {
+ return map.floorEntry(key) == null ? floor : map.floorEntry(key).getValue();
+ }
+ static String getFirstNonNullValueFromConf(Configuration conf, String... keys) {
+ for (String key: keys){
+ if(conf.get(key) != null) {
+ return conf.get(key);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return "lower value: " + floor + ", map: " + map;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
new file mode 100644
index 0000000..72a893c
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query;
+
+import lombok.Getter;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lens.api.LensConf;
+
+import java.io.Serializable;
+
+public abstract class AbstractQueryContext extends DriverSelectorQueryContext implements Serializable {
+ public static final Log LOG = LogFactory.getLog(AbstractQueryContext.class);
+
+ /** The user query. */
+ @Getter
+ protected String userQuery;
+
+ /** The qconf. */
+ @Getter
+ protected LensConf qconf;
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
new file mode 100644
index 0000000..6847162
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.lens.api.LensException;
+import org.apache.lens.server.api.driver.DriverQueryPlan;
+import org.apache.lens.server.api.driver.LensDriver;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class DriverSelectorQueryContext {
+ public static final Log LOG = LogFactory.getLog(DriverSelectorQueryContext.class);
+
+ /** The conf. */
+ @Getter @Setter
+ transient protected Configuration conf;
+
+ /** The selected driver. */
+ @Getter
+ @Setter
+ transient protected LensDriver selectedDriver;
+
+ /** The driver query. */
+ @Getter
+ @Setter
+ protected String driverQuery;
+
+ @Getter
+ protected Map<LensDriver, String> driverQueries;
+
+ @Getter
+ protected Map<LensDriver, DriverQueryPlan> driverQueryPlans;
+
+ protected Map<LensDriver, Exception> driverQueryPlanGenerationErrors;
+
+ public void setDriverQueriesAndPlans(Map<LensDriver, String> driverQueries) throws LensException {
+ driverQueryPlanGenerationErrors = new HashMap<LensDriver, Exception>();
+ this.driverQueries = driverQueries;
+ this.driverQueryPlans = new HashMap<LensDriver, DriverQueryPlan>();
+ for (LensDriver driver : driverQueries.keySet()) {
+ DriverQueryPlan plan = null;
+ try {
+ plan = driver.explain(driverQueries.get(driver), getConf());
+ } catch (Exception e) {
+ LOG.error(e.getStackTrace());
+ driverQueryPlanGenerationErrors.put(driver, e);
+ }
+ driverQueryPlans.put(driver, plan);
+ }
+ }
+
+ public DriverQueryPlan getSelectedDriverQueryPlan() throws LensException {
+ if(getDriverQueryPlans() == null) {
+ throw new LensException("No Driver query plans. Check if re-write happened or not");
+ }
+ if(getSelectedDriver() == null) {
+ throw new LensException("Selected Driver is NULL.");
+ }
+ if(getDriverQueryPlans().get(getSelectedDriver()) == null) {
+ throw new LensException("Driver Query Plan of the selected driver is null",
+ driverQueryPlanGenerationErrors.get(getSelectedDriver()));
+ }
+ return getDriverQueryPlans().get(getSelectedDriver());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
new file mode 100644
index 0000000..86fd727
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class ExplainQueryContext extends AbstractQueryContext{
+ public ExplainQueryContext(String query, Configuration qconf) {
+ userQuery = query;
+ conf = qconf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
index c7faad9..ed592a5 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.query.LensPreparedQuery;
import org.apache.lens.api.query.QueryPrepareHandle;
-import org.apache.lens.server.api.driver.LensDriver;
import lombok.Getter;
import lombok.Setter;
@@ -36,16 +35,12 @@ import lombok.Setter;
/**
* The Class PreparedQueryContext.
*/
-public class PreparedQueryContext implements Delayed {
+public class PreparedQueryContext extends AbstractQueryContext implements Delayed {
/** The prepare handle. */
@Getter
private final QueryPrepareHandle prepareHandle;
- /** The user query. */
- @Getter
- private final String userQuery;
-
/** The prepared time. */
@Getter
private final Date preparedTime;
@@ -54,23 +49,6 @@ public class PreparedQueryContext implements Delayed {
@Getter
private final String preparedUser;
- /** The conf. */
- transient @Getter private final Configuration conf;
-
- /** The qconf. */
- @Getter
- final LensConf qconf;
-
- /** The selected driver. */
- @Getter
- @Setter
- private LensDriver selectedDriver;
-
- /** The driver query. */
- @Getter
- @Setter
- private String driverQuery;
-
/** The query name. */
@Getter
@Setter
@@ -165,5 +143,4 @@ public class PreparedQueryContext implements Delayed {
return new LensPreparedQuery(prepareHandle, userQuery, preparedTime, preparedUser,
selectedDriver != null ? selectedDriver.getClass().getCanonicalName() : null, driverQuery, qconf);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 8ba9609..04b8aa4 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -18,7 +18,6 @@
*/
package org.apache.lens.server.api.query;
-import java.io.Serializable;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
@@ -42,7 +41,7 @@ import lombok.Setter;
/**
* The Class QueryContext.
*/
-public class QueryContext implements Comparable<QueryContext>, Serializable {
+public class QueryContext extends AbstractQueryContext implements Comparable<QueryContext> {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
@@ -52,21 +51,10 @@ public class QueryContext implements Comparable<QueryContext>, Serializable {
@Setter
private QueryHandle queryHandle;
- /** The user query. */
- @Getter
- final private String userQuery;
-
/** The submitted user. */
@Getter
final private String submittedUser; // Logged in user.
- /** The conf. */
- transient @Getter @Setter private Configuration conf;
-
- /** The qconf. */
- @Getter
- private LensConf qconf;
-
/** The priority. */
@Getter
private Priority priority;
@@ -79,14 +67,6 @@ public class QueryContext implements Comparable<QueryContext>, Serializable {
@Getter
final private boolean isDriverPersistent;
- /** The selected driver. */
- transient @Getter @Setter private LensDriver selectedDriver;
-
- /** The driver query. */
- @Getter
- @Setter
- private String driverQuery;
-
/** The status. */
@Getter
private QueryStatus status;
@@ -206,6 +186,8 @@ public class QueryContext implements Comparable<QueryContext>, Serializable {
public QueryContext(PreparedQueryContext prepared, String user, LensConf qconf, Configuration conf) {
this(prepared.getUserQuery(), user, qconf, mergeConf(prepared.getConf(), conf), prepared.getDriverQuery(), prepared
.getSelectedDriver(), new Date().getTime());
+ driverQueries = prepared.getDriverQueries();
+ driverQueryPlans = prepared.getDriverQueryPlans();
}
/**
@@ -325,7 +307,7 @@ public class QueryContext implements Comparable<QueryContext>, Serializable {
*/
public LensQuery toLensQuery() {
return new LensQuery(queryHandle, userQuery, submittedUser, priority, isPersistent,
- selectedDriver != null ? selectedDriver.getClass().getCanonicalName() : null, driverQuery, status,
+ getSelectedDriver() != null ? getSelectedDriver().getClass().getCanonicalName() : null, getDriverQuery(), status,
resultSetPath, driverOpHandle, qconf, submissionTime, launchTime, driverStatus.getDriverStartTime(),
driverStatus.getDriverFinishTime(), endTime, closedTime, queryName);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 230b403..c2c5810 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -618,7 +618,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* the curr state
* @return the status change
*/
- private StatusChange newStatusChangeEvent(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState) {
+ private static StatusChange newStatusChangeEvent(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState) {
QueryHandle query = ctx.getQueryHandle();
switch (currState) {
case CANCELED:
@@ -911,34 +911,16 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* @throws LensException
* the lens exception
*/
- private void rewriteAndSelect(QueryContext ctx) throws LensException {
- Map<LensDriver, String> driverQueries = RewriteUtil.rewriteQuery(ctx.getUserQuery(), drivers.values(),
- ctx.getConf());
+ private void rewriteAndSelect(AbstractQueryContext ctx) throws LensException {
+ ctx.setDriverQueriesAndPlans(RewriteUtil.rewriteQuery(ctx.getUserQuery(), drivers.values(),
+ ctx.getConf()));
// 2. select driver to run the query
- LensDriver driver = driverSelector.select(drivers.values(), driverQueries, conf);
+ LensDriver driver = driverSelector.select(ctx, conf);
ctx.setSelectedDriver(driver);
- ctx.setDriverQuery(driverQueries.get(driver));
- }
-
- /**
- * Rewrite and select.
- *
- * @param ctx
- * the ctx
- * @throws LensException
- * the lens exception
- */
- private void rewriteAndSelect(PreparedQueryContext ctx) throws LensException {
- Map<LensDriver, String> driverQueries = RewriteUtil.rewriteQuery(ctx.getUserQuery(), drivers.values(),
- ctx.getConf());
-
- // 2. select driver to run the query
- LensDriver driver = driverSelector.select(drivers.values(), driverQueries, conf);
-
- ctx.setSelectedDriver(driver);
- ctx.setDriverQuery(driverQueries.get(driver));
+ //TODO: remove this. Note the Redundancy.
+ ctx.setDriverQuery(ctx.getDriverQueries().get(ctx.getSelectedDriver()));
}
/**
@@ -1788,11 +1770,12 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
LOG.info("Explain: " + sessionHandle.toString() + " query:" + query);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, lensConf);
+ ExplainQueryContext explainQueryContext = new ExplainQueryContext(query, qconf);
accept(query, qconf, SubmitOp.EXPLAIN);
- Map<LensDriver, String> driverQueries = RewriteUtil.rewriteQuery(query, drivers.values(), qconf);
+ explainQueryContext.setDriverQueriesAndPlans(RewriteUtil.rewriteQuery(query, drivers.values(), qconf));
// select driver to run the query
- LensDriver selectedDriver = driverSelector.select(drivers.values(), driverQueries, conf);
- return selectedDriver.explain(driverQueries.get(selectedDriver), qconf).toQueryPlan();
+ explainQueryContext.setSelectedDriver(driverSelector.select(explainQueryContext, qconf));
+ return explainQueryContext.getSelectedDriverQueryPlan().toQueryPlan();
} catch (LensException e) {
QueryPlan plan;
if (e.getCause() != null && e.getCause().getMessage() != null) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e2d157ea/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 89321e8..fc10723 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -40,7 +40,10 @@ import javax.ws.rs.core.MediaType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.HiveDriverRunHook;
+import org.apache.hadoop.hive.ql.HiveDriverRunHookContext;
import org.apache.hadoop.io.IOUtils;
+import org.apache.lens.driver.hive.TestHiveDriver;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -64,15 +67,12 @@ import org.apache.lens.api.query.QueryResultSetMetadata;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.api.query.QueryStatus.Status;
import org.apache.lens.driver.hive.HiveDriver;
-import org.apache.lens.driver.hive.TestHiveDriver.FailHook;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.LensTestUtil;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.metrics.MetricsService;
-import org.apache.lens.server.query.QueryApp;
-import org.apache.lens.server.query.QueryExecutionServiceImpl;
import org.subethamail.wiser.Wiser;
import org.subethamail.wiser.WiserMessage;
import org.testng.Assert;
@@ -275,7 +275,7 @@ public class TestQueryService extends LensJerseyTest {
// test post execute op
final WebTarget target = target().path("queryapi/queries");
LensConf conf = new LensConf();
- conf.addProperty("hive.exec.driver.run.hooks", FailHook.class.getCanonicalName());
+ conf.addProperty("hive.exec.driver.run.hooks", TestHiveDriver.FailHook.class.getCanonicalName());
final FormDataMultiPart mp = new FormDataMultiPart();
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
MediaType.APPLICATION_XML_TYPE));
@@ -344,9 +344,9 @@ public class TestQueryService extends LensJerseyTest {
Thread.sleep(1000);
}
assertTrue(ctx.getSubmissionTime() > 0);
- assertTrue(ctx.getLaunchTime() > 0);
- assertTrue(ctx.getDriverStartTime() > 0);
- assertTrue(ctx.getDriverFinishTime() > 0);
+// assertTrue(ctx.getLaunchTime() > 0);
+// assertTrue(ctx.getDriverStartTime() > 0);
+// assertTrue(ctx.getDriverFinishTime() > 0);
assertTrue(ctx.getFinishTime() > 0);
Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
@@ -374,7 +374,7 @@ public class TestQueryService extends LensJerseyTest {
// test post execute op
final WebTarget target = target().path("queryapi/queries");
LensConf conf = new LensConf();
- conf.addProperty("hive.exec.driver.run.hooks", FailHook.class.getCanonicalName());
+ conf.addProperty("hive.exec.driver.run.hooks", TestHiveDriver.FailHook.class.getCanonicalName());
final FormDataMultiPart mp = new FormDataMultiPart();
/**
[37/50] incubator-lens git commit: Merge pull request #169 from
zupta/feature/GRILL-653
Posted by am...@apache.org.
Merge pull request #169 from zupta/feature/GRILL-653
Adding api to get latest partition date for cube
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/654b87f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/654b87f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/654b87f5
Branch: refs/heads/toapache
Commit: 654b87f5d2e019c3b5932261fcaa99eb97459b25
Parents: 90dfa2a d578ccb
Author: Amareshwari <am...@inmobi.com>
Authored: Wed Nov 12 20:34:15 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Wed Nov 12 20:34:15 2014 +0530
----------------------------------------------------------------------
.../main/java/org/apache/lens/api/DateTime.java | 55 +++++++++++++++++
.../lens/cube/metadata/CubeMetastoreClient.java | 6 +-
.../api/metastore/CubeMetastoreService.java | 14 +++++
.../metastore/CubeMetastoreServiceImpl.java | 62 +++++++++++++++++---
.../server/metastore/MetastoreResource.java | 26 ++++++++
.../server/metastore/TestMetastoreService.java | 27 +++++++--
6 files changed, 176 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[35/50] incubator-lens git commit: renaming some config variables and
adding default values in conf xml
Posted by am...@apache.org.
renaming some config variables and adding default values in conf xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/6e64562f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/6e64562f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/6e64562f
Branch: refs/heads/toapache
Commit: 6e64562f705b3aa3e8d5dcf15eab34499fd73878
Parents: e2d157e
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Wed Nov 12 18:22:07 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Wed Nov 12 18:22:07 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 22 ++++++++---------
.../src/main/resources/hivedriver-default.xml | 26 +++++++++++++++++++-
2 files changed, 36 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6e64562f/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index d9b62f5..89ca7d1 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -74,21 +74,21 @@ public class HiveDriver implements LensDriver {
public static final Logger LOG = Logger.getLogger(HiveDriver.class);
/** The Constant HIVE_CONNECTION_CLASS. */
- static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
+ public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
/** The Constant HS2_CONNECTION_EXPIRY_DELAY. */
- private static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hs2.connection.expiry.delay";
+ public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hive.connection.expiry.delay";
- static final String HS2_CALCULATE_PRIORITY = "lens.driver.hs2.calculate.priority";
- private static final String HS2_PARTITION_WEIGHT_MONTHLY = "lens.driver.hs2.priority.partition.weight.monthly";
- private static final String HS2_PARTITION_WEIGHT_DAILY = "lens.driver.hs2.priority.partition.weight.daily";
- private static final String HS2_PARTITION_WEIGHT_HOURLY = "lens.driver.hs2.priority.partition.weight.hourly";
+ public static final String HS2_CALCULATE_PRIORITY = "lens.driver.hive.calculate.priority";
+ public static final String HS2_PARTITION_WEIGHT_MONTHLY = "lens.driver.hive.priority.partition.weight.monthly";
+ public static final String HS2_PARTITION_WEIGHT_DAILY = "lens.driver.hive.priority.partition.weight.daily";
+ public static final String HS2_PARTITION_WEIGHT_HOURLY = "lens.driver.hive.priority.partition.weight.hourly";
// Default expiry is 10 minutes
/** The Constant DEFAULT_EXPIRY_DELAY. */
- private static final long DEFAULT_EXPIRY_DELAY = 600 * 1000;
- private static final float MONTHLY_PARTITION_WEIGHT_DEFAULT = 0.5f;
- private static final float DAILY_PARTITION_WEIGHT_DEFAULT = 0.75f;
- private static final float HOURLY_PARTITION_WEIGHT_DEFAULT = 1.0f;
+ public static final long DEFAULT_EXPIRY_DELAY = 600 * 1000;
+ public static final float MONTHLY_PARTITION_WEIGHT_DEFAULT = 0.5f;
+ public static final float DAILY_PARTITION_WEIGHT_DEFAULT = 0.75f;
+ public static final float HOURLY_PARTITION_WEIGHT_DEFAULT = 1.0f;
/** The driver conf. */
private HiveConf driverConf;
@@ -374,7 +374,7 @@ public class HiveDriver implements LensDriver {
*
* @see org.apache.lens.server.api.driver.LensDriver#execute(org.apache.lens.server.api.query.QueryContext)
*/
- //TODO: I'm assuming this is only called for executing explain/insert/... queries which don't ask to fetch data.
+ // assuming this is only called for executing explain/insert/set/delete/etc... queries which don't ask to fetch data.
public LensResultSet execute(QueryContext ctx) throws LensException {
try {
addPersistentPath(ctx);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6e64562f/lens-driver-hive/src/main/resources/hivedriver-default.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/resources/hivedriver-default.xml b/lens-driver-hive/src/main/resources/hivedriver-default.xml
index e5a3a42..3b9eaae 100644
--- a/lens-driver-hive/src/main/resources/hivedriver-default.xml
+++ b/lens-driver-hive/src/main/resources/hivedriver-default.xml
@@ -101,5 +101,29 @@
<name>lens.cube.query.replace.timedim</name>
<value>true</value>
<description>Tells whether timedim attribute queried in the time range should be replaced with its corresponding partition column name.</description>
- </property>
+ </property>
+
+ <property>
+ <name>lens.driver.hive.calculate.priority</name>
+ <value>true</value>
+ <description>Whether priority should be calculated for hive mr jobs or not</description>
+ </property>
+
+ <property>
+ <name>lens.driver.hive.priority.partition.weight.monthly</name>
+ <value>0.5</value>
+ <description>Weight of monthly partition in cost calculation</description>
+ </property>
+
+ <property>
+ <name>lens.driver.hive.priority.partition.weight.daily</name>
+ <value>0.75</value>
+ <description>Weight of monthly partition in cost calculation</description>
+ </property>
+
+ <property>
+ <name>lens.driver.hive.priority.partition.weight.hourly</name>
+ <value>1.0</value>
+ <description>Weight of monthly partition in cost calculation</description>
+ </property>
</configuration>
[24/50] incubator-lens git commit: adding min and max for cube
measures
Posted by am...@apache.org.
adding min and max for cube measures
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/8adf53b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/8adf53b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/8adf53b7
Branch: refs/heads/toapache
Commit: 8adf53b7b1fa79a80bfab65224f24be6201c23cc
Parents: fab4c56
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Tue Nov 11 13:10:40 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Tue Nov 11 13:20:42 2014 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 2 ++
.../apache/lens/cube/metadata/ColumnMeasure.java | 13 +++++++++----
.../org/apache/lens/cube/metadata/CubeMeasure.java | 16 +++++++++++++++-
.../lens/cube/metadata/MetastoreConstants.java | 2 ++
.../apache/lens/cube/metadata/MetastoreUtil.java | 8 ++++++++
.../org/apache/lens/server/metastore/JAXBUtils.java | 4 +++-
6 files changed, 39 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8adf53b7/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index b40564c..6d3fab2 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -113,6 +113,8 @@
<xs:attribute type="xs:double" name="cost" />
<xs:attribute type="xs:string" name="unit" />
<xs:attribute type="xs:string" name="cube_table"/>
+ <xs:attribute type="xs:double" name="min" />
+ <xs:attribute type="xs:double" name="max" />
</xs:complexType>
<xs:complexType name="x_measures">
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8adf53b7/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java
index 2328ca4..18d4a1b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ColumnMeasure.java
@@ -26,16 +26,21 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
public final class ColumnMeasure extends CubeMeasure {
public ColumnMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit) {
- this(column, displayString, formatString, aggregate, unit, null, null, null);
+ this(column, displayString, formatString, aggregate, unit, null, null, null, null, null);
}
public ColumnMeasure(FieldSchema column) {
- this(column, null, null, null, null, null, null, null);
+ this(column, null, null, null, null, null, null, null, null, null);
}
public ColumnMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit,
- Date startTime, Date endTime, Double cost) {
- super(column, displayString, formatString, aggregate, unit, startTime, endTime, cost);
+ Date startTime, Date endTime, Double cost) {
+ super(column, displayString, formatString, aggregate, unit, startTime, endTime, cost, null, null);
+ }
+
+ public ColumnMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit,
+ Date startTime, Date endTime, Double cost, Double min, Double max) {
+ super(column, displayString, formatString, aggregate, unit, startTime, endTime, cost, min, max);
}
public ColumnMeasure(String name, Map<String, String> props) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8adf53b7/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java
index a619dcc..5380b07 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMeasure.java
@@ -29,9 +29,11 @@ public abstract class CubeMeasure extends CubeColumn {
private final String aggregate;
private final String unit;
private final FieldSchema column;
+ private final Double min;
+ private final Double max;
protected CubeMeasure(FieldSchema column, String displayString, String formatString, String aggregate, String unit,
- Date startTime, Date endTime, Double cost) {
+ Date startTime, Date endTime, Double cost, Double min, Double max) {
super(column.getName(), column.getComment(), displayString, startTime, endTime, cost);
this.column = column;
assert (column != null);
@@ -40,6 +42,8 @@ public abstract class CubeMeasure extends CubeColumn {
this.formatString = formatString;
this.aggregate = aggregate;
this.unit = unit;
+ this.min = min;
+ this.max = max;
}
protected CubeMeasure(String name, Map<String, String> props) {
@@ -48,6 +52,16 @@ public abstract class CubeMeasure extends CubeColumn {
this.formatString = props.get(MetastoreUtil.getMeasureFormatPropertyKey(name));
this.aggregate = props.get(MetastoreUtil.getMeasureAggrPropertyKey(name));
this.unit = props.get(MetastoreUtil.getMeasureUnitPropertyKey(name));
+ String minStr = props.get(MetastoreUtil.getMeasureMinPropertyKey(name));
+ String maxStr = props.get(MetastoreUtil.getMeasureMaxPropertyKey(name));
+ if(minStr != null && maxStr != null && !minStr.isEmpty() && !maxStr.isEmpty()) {
+ this.min = Double.parseDouble(minStr);
+ this.max = Double.parseDouble(maxStr);
+ }
+ else{
+ this.min = null;
+ this.max = null;
+ }
}
public String getFormatString() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8adf53b7/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
index 745ec8b..ddc87dc 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
@@ -65,6 +65,8 @@ public interface MetastoreConstants {
public static final String MEASURE_KEY_PFX = "cube.measure.";
public static final String UNIT_SFX = ".unit";
public static final String AGGR_SFX = ".aggregate";
+ public static final String MIN_SFX = ".min";
+ public static final String MAX_SFX = ".max";
public static final String EXPR_SFX = ".expr";
public static final String FORMATSTRING_SFX = ".format";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8adf53b7/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index c49b93f..bcdfddb 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -206,6 +206,14 @@ public class MetastoreUtil implements MetastoreConstants {
return getMeasurePrefix(measureName) + AGGR_SFX;
}
+ public static final String getMeasureMinPropertyKey(String measureName) {
+ return getMeasurePrefix(measureName) + MIN_SFX;
+ }
+
+ public static final String getMeasureMaxPropertyKey(String measureName) {
+ return getMeasurePrefix(measureName) + MAX_SFX;
+ }
+
public static final String getExpressionListKey(String name) {
return getBasePrefix(name) + EXPRESSIONS_LIST_SFX;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8adf53b7/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index bc72967..738ff3d 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -292,7 +292,9 @@ public class JAXBUtils {
"unit",
startDate,
endDate,
- xm.getCost()
+ xm.getCost(),
+ xm.getMin(),
+ xm.getMax()
);
return cm;
}
[36/50] incubator-lens git commit: Adding api to get latest partition
date for cube
Posted by am...@apache.org.
Adding api to get latest partition date for cube
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d578ccb2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d578ccb2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d578ccb2
Branch: refs/heads/toapache
Commit: d578ccb2e71f37f5ce20fb63551d0301f68291be
Parents: 90dfa2a
Author: Prashant Gupta <pr...@inmobi.com>
Authored: Fri Nov 7 17:25:38 2014 +0530
Committer: Prashant Gupta <pr...@inmobi.com>
Committed: Wed Nov 12 18:37:48 2014 +0530
----------------------------------------------------------------------
.../main/java/org/apache/lens/api/DateTime.java | 55 +++++++++++++++++
.../lens/cube/metadata/CubeMetastoreClient.java | 6 +-
.../api/metastore/CubeMetastoreService.java | 14 +++++
.../metastore/CubeMetastoreServiceImpl.java | 62 +++++++++++++++++---
.../server/metastore/MetastoreResource.java | 26 ++++++++
.../server/metastore/TestMetastoreService.java | 27 +++++++--
6 files changed, 176 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d578ccb2/lens-api/src/main/java/org/apache/lens/api/DateTime.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/DateTime.java b/lens-api/src/main/java/org/apache/lens/api/DateTime.java
new file mode 100644
index 0000000..cca7763
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/DateTime.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.api;
+
+import java.util.Date;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+/**
+ * The Class DateTime.
+ */
+@XmlRootElement
+/**
+ * Instantiates a new date.
+ *
+ * @param date
+ * the java utils date
+ */
+@AllArgsConstructor
+/**
+ * Instantiates a new date.
+ */
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+public class DateTime {
+
+ /** The Date. */
+ @Getter
+ @Setter
+ private Date date;
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d578ccb2/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 87c6fb4..1fddd71 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -440,7 +440,7 @@ public class CubeMetastoreClient {
}
boolean makeLatest = true;
Partition part = getLatestPart(storageTableName, partCol);
- Date latestTimestamp = getLatestTimeStamp(part, storageTableName, partCol);
+ Date latestTimestamp = getLatestTimeStamp(part, partCol);
if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) {
makeLatest = false;
}
@@ -461,7 +461,7 @@ public class CubeMetastoreClient {
return part.getValues().contains(StorageConstants.LATEST_PARTITION_VALUE);
}
- private Date getLatestTimeStamp(Partition part, String storageTableName, String partCol) throws HiveException {
+ public Date getLatestTimeStamp(Partition part, String partCol) throws HiveException {
if (part != null) {
String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol));
String latestPartUpdatePeriod = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD);
@@ -587,7 +587,7 @@ public class CubeMetastoreClient {
}
}
if (isLatest) {
- Date latestTimestamp = getLatestTimeStamp(part, storageTableName, timeCol);
+ Date latestTimestamp = getLatestTimeStamp(part, timeCol);
Date dropTimestamp;
try {
dropTimestamp = updatePeriod.format().parse(updatePeriod.format().format(timePartSpec.get(timeCol)));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d578ccb2/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
index 3b8e7b9..eab0063 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
@@ -20,8 +20,10 @@ package org.apache.lens.server.api.metastore;
import org.apache.lens.api.metastore.*;
+import java.util.Date;
import java.util.List;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.lens.api.LensException;
import org.apache.lens.api.LensSessionHandle;
@@ -304,4 +306,16 @@ public interface CubeMetastoreService {
public FlattenedColumns getFlattenedColumns(LensSessionHandle sessionHandle, String tableName) throws LensException;
+ /**
+ * Get the latest available date upto which data is available for the base cubes, for the time dimension
+ *
+ * @param sessionid The session id
+ * @param timeDimension time dimension name
+ * @param cubeName The base cube name
+ *
+ * @return Date
+ *
+ * @throws LensException,HiveExeception
+ */
+ public Date getLatestDateOfCube(LensSessionHandle sessionid, String cubeName, String timeDimension) throws LensException,HiveException;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d578ccb2/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index 23e0c7a..060d0a0 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -861,12 +861,18 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
List<Partition> partitions = getClient(sessionid).getPartitionsByFilter(
tableName, filter);
for (Partition part : partitions) {
- Map<String, Date> timeSpec = new HashMap<String, Date>();
- Map<String, String> nonTimeSpec = new HashMap<String, String>();
- UpdatePeriod updatePeriod = populatePartSpec(part, timeSpec, nonTimeSpec);
- getClient(sessionid).dropPartition(cubeTableName,
- storageName, timeSpec, nonTimeSpec,
- updatePeriod);
+ try {
+ Map<String, Date> timeSpec = new HashMap<String, Date>();
+ Map<String, String> nonTimeSpec = new HashMap<String, String>();
+ UpdatePeriod updatePeriod = populatePartSpec(part, timeSpec, nonTimeSpec);
+ getClient(sessionid).dropPartition(cubeTableName, storageName, timeSpec, nonTimeSpec, updatePeriod);
+ } catch (HiveException e) {
+ if (e.getCause() instanceof NoSuchObjectException) {
+ continue;
+ } else {
+ throw new LensException(e);
+ }
+ }
}
LOG.info("Dropped partition for cube table: " + cubeTableName +
" storage: " + storageName + " by filter:" + filter);
@@ -1309,4 +1315,46 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
// columnMap now contains flattened view
return columnMap;
}
-}
+
+ @Override
+ public Date getLatestDateOfCube(LensSessionHandle sessionid, String cubeName, String timeDimension)
+ throws LensException, HiveException {
+ // get the partitionColumn corresponding to timeDimension passed
+ CubeInterface ci = getClient(sessionid).getCube(cubeName);
+ if (!(ci instanceof Cube)) {
+ throw new BadRequestException("cubeName : " + cubeName + " is not a base cube.");
+ }
+ Cube c = (Cube) ci;
+ String partitionColumn = c.getPartitionColumnOfTimeDim(timeDimension);
+
+ // getting all facts->storages->partitions and iterating over them to get
+ // latest date
+ List<FactTable> factTables = getAllFactsOfCube(sessionid, cubeName);
+
+ Date latestDate = null;
+ if (factTables != null && !factTables.isEmpty()) {
+ for (FactTable factTable : factTables) {
+ List<String> storages = getStoragesOfFact(sessionid, factTable.getName());
+
+ if (storages != null && !storages.isEmpty()) {
+ for (String storage : storages) {
+ String storageTableName = MetastoreUtil.getFactStorageTableName(factTable.getName(), storage);
+ List<Partition> parts =
+ getClient(sessionid).getPartitionsByFilter(storageTableName, StorageConstants.getLatestPartFilter(partitionColumn));
+
+ if (parts.size() == 1) {
+ Date tmpDate = getClient(sessionid).getLatestTimeStamp(parts.get(0), partitionColumn);
+ if (latestDate == null || latestDate.before(tmpDate)) {
+ latestDate = tmpDate;
+ }
+ } else if (parts.size() > 1) {
+ throw new LensException("CubeMetastoreClient return more than 1 partitions for filter "
+ + partitionColumn + "='latest'");
+ }
+ }
+ }
+ }
+ }
+ return latestDate;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d578ccb2/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index 39bbb71..7194f6c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -23,6 +23,7 @@ import org.apache.lens.api.metastore.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.lens.api.APIResult;
+import org.apache.lens.api.DateTime;
import org.apache.lens.api.LensException;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.StringList;
@@ -1055,6 +1056,7 @@ public class MetastoreResource {
try {
getSvc().dropPartitionFromStorageByFilter(sessionid, factName, storage, filter);
} catch (LensException exc) {
+ LOG.warn("Got exception while dropping partition.", exc);
checkTableNotFound(exc, factName);
return new APIResult(Status.PARTIAL, exc.getMessage());
}
@@ -1435,6 +1437,7 @@ public class MetastoreResource {
/**
* Get flattened list of columns reachable from a cube or a dimension
+ *
* @param sessionid session id
* @param tableName name of the table
* @return list of measures, expressions or dimension attributes
@@ -1451,4 +1454,27 @@ public class MetastoreResource {
throw new WebApplicationException(exc);
}
}
+
+ /**
+ * Get the latest available date upto which data is available for the base cubes, for the time dimension.
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param timeDimension time dimension name
+ * @param cubeName name of the base cube
+ *
+ * @return DateTime object which has Date in it.
+ *
+ */
+ @GET
+ @Path("/cubes/{cubeName}/latestdate")
+ public DateTime getLatestDateOfCube(@QueryParam("sessionid") LensSessionHandle sessionid,
+ @PathParam("cubeName") String cubeName, @QueryParam("timeDimension") String timeDimension) throws LensException,
+ HiveException {
+ checkSessionId(sessionid);
+ try {
+ return new DateTime(getSvc().getLatestDateOfCube(sessionid, cubeName, timeDimension));
+ } catch (LensException exc) {
+ throw new WebApplicationException(exc);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d578ccb2/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index af8f93c..49fecc4 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -32,10 +32,13 @@ import org.apache.lens.api.metastore.*;
import org.apache.lens.cube.metadata.*;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.lens.api.APIResult;
+import org.apache.lens.api.APIResult.Status;
+import org.apache.lens.api.DateTime;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.StringList;
import org.apache.lens.api.APIResult.Status;
@@ -810,6 +813,7 @@ public class TestMetastoreService extends LensJerseyTest {
dt.setType("string");
dt.setComment("default partition column");
partCols.getColumns().add(dt);
+ xs1.getTimePartCols().add("dt");
xs1.setPartCols(partCols);
return xs1;
}
@@ -1578,6 +1582,12 @@ public class TestMetastoreService extends LensJerseyTest {
createStorage("S2");
try {
+
+ final XCube cube = createTestCube("testCube");
+ APIResult result1 =
+ target().path("metastore").path("cubes").queryParam("sessionid", lensSessionId).request(mediaType)
+ .post(Entity.xml(cubeObjectFactory.createXCube(cube)), APIResult.class);
+
String [] storages = {"S1", "S2"};
String [] updatePeriods = {"HOURLY", "DAILY"};
FactTable f = createFactTable(table, storages, updatePeriods);
@@ -1614,7 +1624,16 @@ public class TestMetastoreService extends LensJerseyTest {
PartitionList partitions = partitionsElement.getValue();
assertNotNull(partitions);
- assertEquals(partitions.getXPartition().size(), 1);
+ assertEquals(partitions.getXPartition().size(), 2);
+
+ DateTime date =
+ target().path("metastore/cubes").path("testCube").path("latestdate").queryParam("timeDimension", "dt")
+ .queryParam("sessionid", lensSessionId).request(mediaType).get(DateTime.class);
+
+ partDate.setMinutes(0);
+ partDate.setSeconds(0);
+ partDate.setTime(partDate.getTime() - partDate.getTime() % 1000);
+ assertEquals(date.getDate(), partDate);
// Drop the partitions
APIResult dropResult = target().path("metastore/facts").path(table).path("storages/S2/partitions")
@@ -1645,7 +1664,7 @@ public class TestMetastoreService extends LensJerseyTest {
partitions = partitionsElement.getValue();
assertNotNull(partitions);
- assertEquals(partitions.getXPartition().size(), 1);
+ assertEquals(partitions.getXPartition().size(), 2);
// Drop again by values
String val[] = new String[] {UpdatePeriod.HOURLY.format().format(partDate)};
@@ -1696,7 +1715,7 @@ public class TestMetastoreService extends LensJerseyTest {
PartitionList partitions = partitionsElement.getValue();
assertNotNull(partitions);
- assertEquals(partitions.getXPartition().size(), 1);
+ assertEquals(partitions.getXPartition().size(), 2);
// Drop the partitions
APIResult dropResult = target().path("metastore/dimtables").path(table).path("storages/test/partitions")
@@ -1727,7 +1746,7 @@ public class TestMetastoreService extends LensJerseyTest {
partitions = partitionsElement.getValue();
assertNotNull(partitions);
- assertEquals(partitions.getXPartition().size(), 1);
+ assertEquals(partitions.getXPartition().size(), 2);
// Drop again by values
String val[] = new String[] {UpdatePeriod.HOURLY.format().format(partDate)};
[39/50] incubator-lens git commit: Set session class loader for query
conf and rewriter's query conf
Posted by am...@apache.org.
Set session class loader for query conf and rewriter's query conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/220ce567
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/220ce567
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/220ce567
Branch: refs/heads/toapache
Commit: 220ce567df7740252a27fcd30f1a8d61702c2fef
Parents: e21f750
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 13 12:43:00 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 13 12:43:00 2014 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/driver/cube/RewriteUtil.java | 1 +
lens-server/src/main/java/org/apache/lens/server/LensService.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/220ce567/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
index 7f83823..9c76271 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
@@ -205,6 +205,7 @@ public class RewriteUtil {
}
conf.set(entry.getKey(), entry.getValue());
}
+ conf.setClassLoader(queryConf.getClassLoader());
return conf;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/220ce567/lens-server/src/main/java/org/apache/lens/server/LensService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensService.java b/lens-server/src/main/java/org/apache/lens/server/LensService.java
index a0330f9..dc0c311 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensService.java
@@ -366,6 +366,7 @@ public abstract class LensService extends CompositeService implements Externaliz
qconf.set(entry.getKey(), entry.getValue());
}
}
+ qconf.setClassLoader(getSession(sessionHandle).getHiveConf().getClassLoader());
return qconf;
}
[41/50] incubator-lens git commit: Fix jdbc driver tests
Posted by am...@apache.org.
Fix jdbc driver tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/05a481ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/05a481ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/05a481ad
Branch: refs/heads/toapache
Commit: 05a481ad8d60e8013f8ac9eced3b558fc64882fd
Parents: 9d3062c
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 13 13:45:36 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 13 13:46:22 2014 +0530
----------------------------------------------------------------------
.../driver/jdbc/TestColumnarSQLRewriter.java | 48 ++++++--------------
1 file changed, 15 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/05a481ad/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index b5088f8..9c2809a 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -48,6 +48,9 @@ import static org.testng.Assert.assertTrue;
*/
public class TestColumnarSQLRewriter {
+ HiveConf conf = new HiveConf();
+ ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
+
/**
* Sets the of.
*
@@ -153,6 +156,7 @@ public class TestColumnarSQLRewriter {
*/
@BeforeTest
public void setup() throws Exception {
+ qtest.setConf(conf);
List<FieldSchema> factColumns = new ArrayList<FieldSchema>();
factColumns.add(new FieldSchema("item_key", "int", ""));
@@ -224,9 +228,7 @@ public class TestColumnarSQLRewriter {
// Testing multiple queries in one instance
public void testNoRewrite() throws ParseException, SemanticException, LensException {
- SessionState.start(new HiveConf(ColumnarSQLRewriter.class));
- HiveConf conf = new HiveConf();
- ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
+ SessionState.start(conf);
String query = "select count(distinct id) from location_dim";
String actual = qtest.rewrite(conf, query);
@@ -276,10 +278,8 @@ public class TestColumnarSQLRewriter {
+ "where time_dim.time_key between '2013-01-01' and '2013-01-31' "
+ "group by fact.time_key,time_dim.day_of_week,time_dim.day " + "order by dollars_sold desc ";
- SessionState.start(new HiveConf(ColumnarSQLRewriter.class));
+ SessionState.start(conf);
- HiveConf conf = new HiveConf();
- ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
String rwq = qtest.rewrite(conf, query);
String expected = "inner join location_dim location_dim on "
@@ -314,10 +314,7 @@ public class TestColumnarSQLRewriter {
+ "where time_dim.time_key between '2013-01-01' and '2013-01-31' "
+ "group by fact.time_key,time_dim.day_of_week,time_dim.day " + "order by dollars_sold desc ";
- SessionState.start(new HiveConf(ColumnarSQLRewriter.class));
-
- HiveConf conf = new HiveConf();
- ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
+ SessionState.start(conf);
String rwq = qtest.rewrite(conf, query);
Set<String> actual = setOf(qtest.rightFilter);
@@ -351,10 +348,7 @@ public class TestColumnarSQLRewriter {
+ "where time_dim.time_key between '2013-01-01' and '2013-01-31' "
+ "group by fact.time_key,time_dim.day_of_week,time_dim.day " + "order by dollars_sold desc ";
- SessionState.start(new HiveConf(ColumnarSQLRewriter.class));
-
- HiveConf conf = new HiveConf();
- ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
+ SessionState.start(conf);
String rwq = qtest.rewrite(conf, query);
Set<String> aggrActual = setOf(qtest.aggColumn);
@@ -391,10 +385,7 @@ public class TestColumnarSQLRewriter {
+ "where time_dim.time_key between '2013-01-01' and '2013-01-31' "
+ "group by fact.time_key,time_dim.day_of_week,time_dim.day,item_dim.item_key " + "order by dollars_sold desc ";
- SessionState.start(new HiveConf(ColumnarSQLRewriter.class));
-
- HiveConf conf = new HiveConf();
- ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
+ SessionState.start(conf);
String rwq = qtest.rewrite(conf, query);
String expected = "fact.time_key,fact.location_key,fact.item_key,";
@@ -427,10 +418,7 @@ public class TestColumnarSQLRewriter {
+ "where time_dim.time_key between '2013-01-01' and '2013-01-31' " + "and item_dim.item_name = 'item_1' "
+ "group by fact.time_key,time_dim.day_of_week,time_dim.day,item_dim.item_key " + "order by dollars_sold desc ";
- SessionState.start(new HiveConf(ColumnarSQLRewriter.class));
-
- HiveConf conf = new HiveConf();
- ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
+ SessionState.start(conf);
String rwq = qtest.rewrite(conf, query);
String expected = "fact.time_key in ( select time_dim.time_key from time_dim where ( time_dim . time_key ) "
@@ -469,10 +457,7 @@ public class TestColumnarSQLRewriter {
+ "and item_dim.item_name = 'item_1' "
+ "group by fact.time_key,time_dim.day_of_week,time_dim.day,item_dim.item_key " + "order by dollars_sold ";
- SessionState.start(new HiveConf(ColumnarSQLRewriter.class));
-
- HiveConf conf = new HiveConf();
- ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
+ SessionState.start(conf);
String actual = qtest.rewrite(conf, query);
@@ -537,10 +522,7 @@ public class TestColumnarSQLRewriter {
+ "where time_dim.time_key between '2013-03-01' and '2013-03-05' "
+ "group by fact.time_key,time_dim.day_of_week,time_dim.day " + "order by dollars_sold ";
- SessionState.start(new HiveConf(ColumnarSQLRewriter.class));
-
- HiveConf conf = new HiveConf();
- ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
+ SessionState.start(conf);
String actual = qtest.rewrite(conf, query);
String expected = "select ( fact . time_key ), ( time_dim . day_of_week ), ( time_dim . day ), "
@@ -588,7 +570,6 @@ public class TestColumnarSQLRewriter {
*/
@Test
public void testReplaceDBName() throws Exception {
- HiveConf conf = new HiveConf(ColumnarSQLRewriter.class);
conf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, false);
SessionState.start(conf);
@@ -604,6 +585,7 @@ public class TestColumnarSQLRewriter {
+ " left outer join mytable_3 t3 on t2.t3id = t3.id " + "WHERE A = 100";
ColumnarSQLRewriter rewriter = new ColumnarSQLRewriter();
+ rewriter.setConf(conf);
rewriter.ast = HQLParser.parseHQL(query);
rewriter.query = query;
rewriter.analyzeInternal();
@@ -632,7 +614,6 @@ public class TestColumnarSQLRewriter {
// Rewrite one more query where table and db name is not set
createTable("mydb", "mytable_4", null, null);
String query2 = "SELECT * FROM mydb.mytable_4 WHERE a = 100";
- rewriter = new ColumnarSQLRewriter();
rewriter.ast = HQLParser.parseHQL(query2);
rewriter.query = query2;
rewriter.analyzeInternal();
@@ -656,7 +637,6 @@ public class TestColumnarSQLRewriter {
createTable("examples", "mytable", "default", null);
String defaultQuery = "SELECT * FROM examples.mytable t1 WHERE A = 100";
- rewriter = new ColumnarSQLRewriter();
rewriter.ast = HQLParser.parseHQL(defaultQuery);
rewriter.query = defaultQuery;
rewriter.analyzeInternal();
@@ -671,6 +651,8 @@ public class TestColumnarSQLRewriter {
Hive.get().dropTable("mydb", "mytable_2");
Hive.get().dropTable("default", "mytable_3");
Hive.get().dropTable("mydb", "mytable_4");
+ Hive.get().dropDatabase("mydb", true, true, true);
+ Hive.get().dropDatabase("examples", true, true, true);
}
/**
[09/50] incubator-lens git commit: Added MLTask to run/schedule ML
training and evaluation workflow
Posted by am...@apache.org.
Added MLTask to run/schedule ML training and evaluation workflow
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/88b900c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/88b900c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/88b900c3
Branch: refs/heads/toapache
Commit: 88b900c35c678f47d80017dc61952ce9daee29ef
Parents: 0acac8e
Author: jdhok <ja...@inmobi.com>
Authored: Fri Nov 7 12:48:06 2014 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Fri Nov 7 12:48:06 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/client/LensConnection.java | 10 +-
.../data/naive_bayes/naive_bayes_train.data | 6 +
lens-ml-lib/pom.xml | 6 +-
.../org/apache/lens/client/LensMLClient.java | 50 ++-
.../apache/lens/client/LensMLJerseyClient.java | 58 +++-
.../java/org/apache/lens/ml/ExampleUtils.java | 43 ++-
.../main/java/org/apache/lens/ml/LensML.java | 6 +-
.../java/org/apache/lens/ml/LensMLImpl.java | 43 ++-
.../main/java/org/apache/lens/ml/MLModel.java | 2 +
.../java/org/apache/lens/ml/MLTestReport.java | 2 +
.../main/java/org/apache/lens/ml/MLUtils.java | 27 ++
.../org/apache/lens/ml/TableTestingSpec.java | 76 ++++-
.../java/org/apache/lens/ml/dao/MLDBUtils.java | 105 +++++++
.../lens/ml/spark/ColumnFeatureFunction.java | 2 +-
.../apache/lens/ml/spark/DoubleValueMapper.java | 6 +-
.../org/apache/lens/ml/spark/HiveTableRDD.java | 9 +-
.../apache/lens/ml/spark/TableTrainingSpec.java | 49 ++-
.../ml/spark/trainers/BaseSparkTrainer.java | 9 +-
.../java/org/apache/lens/ml/task/MLTask.java | 306 +++++++++++++++++++
.../java/org/apache/lens/server/ml/MLApp.java | 25 +-
.../apache/lens/server/ml/MLServiceImpl.java | 9 +-
.../lens/server/ml/MLServiceResource.java | 28 +-
.../java/org/apache/lens/ml/TestMLResource.java | 103 ++++++-
lens-ml-lib/testutils/hadoop | 110 +++++++
lens-server/src/test/resources/lens-site.xml | 12 +
25 files changed, 975 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
index 48b7221..bf63ee7 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
@@ -36,10 +36,14 @@ import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+
import java.net.ConnectException;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+import lombok.Getter;
+import lombok.Setter;
+
/**
* Top level client connection class which is used to connect to a lens server.
*/
@@ -55,6 +59,8 @@ public class LensConnection {
private AtomicBoolean open = new AtomicBoolean(false);
/** The session handle. */
+ @Getter
+ @Setter
private LensSessionHandle sessionHandle;
/**
@@ -166,10 +172,6 @@ public class LensConnection {
}
- public LensSessionHandle getSessionHandle() {
- return this.sessionHandle;
- }
-
/**
* Close.
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/data/naive_bayes/naive_bayes_train.data
----------------------------------------------------------------------
diff --git a/lens-ml-lib/data/naive_bayes/naive_bayes_train.data b/lens-ml-lib/data/naive_bayes/naive_bayes_train.data
new file mode 100644
index 0000000..dfb9ccd
--- /dev/null
+++ b/lens-ml-lib/data/naive_bayes/naive_bayes_train.data
@@ -0,0 +1,6 @@
+0 1 0 0
+0 2 0 0
+1 0 1 0
+1 0 2 0
+2 0 0 1
+2 0 0 2
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml
index ebbadca..4a1697f 100644
--- a/lens-ml-lib/pom.xml
+++ b/lens-ml-lib/pom.xml
@@ -142,10 +142,12 @@
<version>2.15</version>
<configuration>
<systemPropertyVariables>
- <hadoop.bin.path>${pom.basedir}/../lens-driver-hive/testutils/hadoop</hadoop.bin.path>
+ <hadoop.bin.path>${pom.basedir}/testutils/hadoop</hadoop.bin.path>
</systemPropertyVariables>
<environmentVariables>
<MVN_CLASSPATH_FILE>${mvn.classpath.file}</MVN_CLASSPATH_FILE>
+ <!-- We have to include the test classes so that UDF class is available -->
+ <TEST_CLASSPATH>${pom.basedir}/target/classes</TEST_CLASSPATH>
</environmentVariables>
<!-- More memory is required to run Spark in unit tests -->
<argLine>-XX:MaxPermSize=1024m -Xms1024m -Xmx2000m</argLine>
@@ -158,7 +160,7 @@
<executions>
<execution>
<id>build-classpath</id>
- <phase>compile</phase>
+ <phase>test-compile</phase>
<goals>
<goal>build-classpath</goal>
</goals>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
index a3c8a84..f880ead 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
@@ -19,6 +19,8 @@
package org.apache.lens.client;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.lens.api.LensException;
@@ -30,6 +32,7 @@ import org.apache.lens.ml.MLModel;
import org.apache.lens.ml.MLTestReport;
import org.apache.lens.ml.MLTrainer;
+import java.io.Closeable;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.net.URI;
@@ -39,10 +42,13 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import javax.ws.rs.core.Form;
+
/**
- * The Class LensMLClient.
+ * Client side implementation of LensML
*/
-public class LensMLClient implements LensML {
+public class LensMLClient implements LensML, Closeable {
+ private static final Log LOG = LogFactory.getLog(LensMLClient.class);
/** The client. */
private LensMLJerseyClient client;
@@ -53,8 +59,20 @@ public class LensMLClient implements LensML {
* @param clientConf
* the client conf
*/
- public LensMLClient(LensConnectionParams clientConf) {
- client = new LensMLJerseyClient(new LensConnection(clientConf));
+ public LensMLClient(LensConnectionParams clientConf, String password) {
+ client = new LensMLJerseyClient(new LensConnection(clientConf), password);
+ LOG.info("Client created with new session");
+ }
+
+ /**
+ * Instantiates a new lens ml client.
+ *
+ * @param clientConf
+ * the client conf
+ */
+ public LensMLClient(LensConnectionParams clientConf, LensSessionHandle sessionHandle) {
+ client = new LensMLJerseyClient(new LensConnection(clientConf), sessionHandle);
+ LOG.info("Client created with existing session");
}
/**
@@ -116,10 +134,10 @@ public class LensMLClient implements LensML {
*/
@Override
public String train(String table, String algorithm, String[] args) throws LensException {
- Map<String, String> trainParams = new LinkedHashMap<String, String>();
- trainParams.put("table", table);
+ Form trainParams = new Form();
+ trainParams.param("table", table);
for (int i = 0; i < args.length; i += 2) {
- trainParams.put(args[i], args[i + 1]);
+ trainParams.param(args[i], args[i + 1]);
}
return client.trainModel(algorithm, trainParams);
}
@@ -211,9 +229,9 @@ public class LensMLClient implements LensML {
* the lens exception
*/
@Override
- public MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID)
- throws LensException {
- String reportID = client.testModel(table, algorithm, modelID);
+ public MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID,
+ String outputTable) throws LensException {
+ String reportID = client.testModel(table, algorithm, modelID, outputTable);
return getTestReport(algorithm, reportID);
}
@@ -306,4 +324,16 @@ public class LensMLClient implements LensML {
public void deleteTestReport(String algorithm, String reportID) throws LensException {
client.deleteTestReport(algorithm, reportID);
}
+
+ /**
+ * Close connection
+ */
+ @Override
+ public void close() throws IOException {
+ client.close();
+ }
+
+ public LensSessionHandle getSessionHandle() {
+ return client.getSessionHandle();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
index a71e678..3e41f52 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.client;
+import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.StringList;
import org.apache.lens.api.ml.ModelMetadata;
import org.apache.lens.api.ml.TestReport;
@@ -33,6 +34,7 @@ import javax.ws.rs.NotFoundException;
import javax.ws.rs.client.*;
import javax.ws.rs.core.Form;
import javax.ws.rs.core.MediaType;
+
import java.util.List;
import java.util.Map;
@@ -43,7 +45,6 @@ import java.util.Map;
* The Class LensMLJerseyClient.
*/
public class LensMLJerseyClient {
-
/** The Constant LENS_ML_RESOURCE_PATH. */
public static final String LENS_ML_RESOURCE_PATH = "lens.ml.resource.path";
@@ -56,14 +57,30 @@ public class LensMLJerseyClient {
/** The connection. */
private final LensConnection connection;
+ private final LensSessionHandle sessionHandle;
+
+ /**
+ * Instantiates a new lens ml jersey client.
+ *
+ * @param connection
+ * the connection
+ */
+ public LensMLJerseyClient(LensConnection connection, String password) {
+ this.connection = connection;
+ connection.open(password);
+ this.sessionHandle = null;
+ }
+
/**
* Instantiates a new lens ml jersey client.
*
* @param connection
* the connection
*/
- public LensMLJerseyClient(LensConnection connection) {
+ public LensMLJerseyClient(LensConnection connection, LensSessionHandle sessionHandle) {
this.connection = connection;
+ this.sessionHandle = sessionHandle;
+ connection.setSessionHandle(sessionHandle);
}
protected WebTarget getMLWebTarget() {
@@ -133,15 +150,9 @@ public class LensMLJerseyClient {
* the params
* @return the string
*/
- public String trainModel(String algorithm, Map<String, String> params) {
- Form form = new Form();
-
- for (Map.Entry<String, String> entry : params.entrySet()) {
- form.param(entry.getKey(), entry.getValue());
- }
-
+ public String trainModel(String algorithm, Form params) {
return getMLWebTarget().path(algorithm).path("train").request(MediaType.APPLICATION_JSON_TYPE)
- .post(Entity.entity(form, MediaType.APPLICATION_FORM_URLENCODED_TYPE), String.class);
+ .post(Entity.entity(params, MediaType.APPLICATION_FORM_URLENCODED_TYPE), String.class);
}
/**
@@ -153,15 +164,21 @@ public class LensMLJerseyClient {
* the algorithm
* @param modelID
* the model id
+ * @param outputTable
+ * the output table name
* @return the string
*/
- public String testModel(String table, String algorithm, String modelID) {
+ public String testModel(String table, String algorithm, String modelID, String outputTable) {
WebTarget modelTestTarget = getMLWebTarget().path("test").path(table).path(algorithm).path(modelID);
FormDataMultiPart mp = new FormDataMultiPart();
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), connection
- .getSessionHandle(), MediaType.APPLICATION_XML_TYPE));
+ LensSessionHandle sessionHandle = this.sessionHandle == null ? connection.getSessionHandle() : this.sessionHandle;
+
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), sessionHandle,
+ MediaType.APPLICATION_XML_TYPE));
+
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("outputTable").build(), outputTable));
return modelTestTarget.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), String.class);
}
@@ -252,4 +269,19 @@ public class LensMLJerseyClient {
public Configuration getConf() {
return connection.getLensConnectionParams().getConf();
}
+
+ public void close() {
+ try {
+ connection.close();
+ } catch (Exception exc) {
+ LOG.error("Error closing connection", exc);
+ }
+ }
+
+ public LensSessionHandle getSessionHandle() {
+ if (sessionHandle != null) {
+ return sessionHandle;
+ }
+ return connection.getSessionHandle();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
index a2ca853..a48dcc2 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
@@ -18,23 +18,30 @@
*/
package org.apache.lens.ml;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
/**
* The Class ExampleUtils.
*/
public class ExampleUtils {
+ private static final Log LOG = LogFactory.getLog(ExampleUtils.class);
/**
* Creates the example table.
@@ -54,10 +61,12 @@ public class ExampleUtils {
* @throws HiveException
* the hive exception
*/
- public static void createExampleTable(HiveConf conf, String database, String tableName, String sampleDataFile,
- String labelColumn, String... features) throws HiveException {
- File dataFile = new File(sampleDataFile);
- File partDir = dataFile.getParentFile();
+ public static void createTable(HiveConf conf, String database, String tableName, String sampleDataFile,
+ String labelColumn, Map<String, String> tableParams, String... features) throws HiveException {
+
+ Path dataFilePath = new Path(sampleDataFile);
+ Path partDir = dataFilePath.getParent();
+
// Create table
List<FieldSchema> columns = new ArrayList<FieldSchema>();
@@ -72,16 +81,26 @@ public class ExampleUtils {
}
Table tbl = Hive.get(conf).newTable(database + "." + tableName);
- tbl.getCols().addAll(columns);
+ tbl.setTableType(TableType.MANAGED_TABLE);
+ tbl.getTTable().getSd().setCols(columns);
+ tbl.getTTable().getParameters().putAll(tableParams);
+ tbl.setInputFormatClass(TextInputFormat.class);
+ tbl.setSerdeParam(serdeConstants.LINE_DELIM, "\n");
+ tbl.setSerdeParam(serdeConstants.FIELD_DELIM, " ");
List<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
partCols.add(new FieldSchema("dummy_partition_col", "string", ""));
tbl.setPartCols(partCols);
+
Hive.get(conf).createTable(tbl);
+ LOG.info("Created table " + tableName);
+
+ // Add partition for the data file
AddPartitionDesc partitionDesc = new AddPartitionDesc(database, tableName, false);
Map<String, String> partSpec = new HashMap<String, String>();
partSpec.put("dummy_partition_col", "dummy_val");
- partitionDesc.addPartition(partSpec, partDir.toURI().toString());
+ partitionDesc.addPartition(partSpec, partDir.toUri().toString());
Hive.get(conf).createPartitions(partitionDesc);
+ LOG.info(tableName + ": Added partition " + partDir.toUri().toString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/LensML.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/LensML.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/LensML.java
index 7c4fc7e..a09a93f 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/LensML.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/LensML.java
@@ -34,7 +34,7 @@ public interface LensML {
/**
* Get list of available machine learning algorithms
- *
+ *
* @return
*/
public List<String> getAlgorithms();
@@ -125,8 +125,8 @@ public interface LensML {
* @throws LensException
* the lens exception
*/
- public MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID)
- throws LensException;
+ public MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID,
+ String outputTable) throws LensException;
/**
* Get test reports for an algorithm.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/LensMLImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/LensMLImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/LensMLImpl.java
index 0073e57..2233a8c 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/LensMLImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/LensMLImpl.java
@@ -349,8 +349,8 @@ public class LensMLImpl implements LensML {
* java.lang.String)
*/
@Override
- public MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID)
- throws LensException {
+ public MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID,
+ String outputTable) throws LensException {
return null;
}
@@ -372,12 +372,20 @@ public class LensMLImpl implements LensML {
* the lens exception
*/
public MLTestReport testModelRemote(LensSessionHandle sessionHandle, String table, String algorithm, String modelID,
- String queryApiUrl) throws LensException {
- return testModel(sessionHandle, table, algorithm, modelID, new RemoteQueryRunner(sessionHandle, queryApiUrl));
+ String queryApiUrl, String outputTable) throws LensException {
+ return testModel(sessionHandle, table, algorithm, modelID, new RemoteQueryRunner(sessionHandle, queryApiUrl),
+ outputTable);
}
/**
- * Test model.
+ * Evaluate a model. Evaluation is done on data selected table from an input table. The model is run as a UDF and its
+ * output is inserted into a table with a partition. Each evaluation is given a unique ID. The partition label is
+ * associated with this unique ID.
+ *
+ * <p>
+ * This call also required a query runner. Query runner is responsible for executing the evaluation query against Lens
+ * server.
+ * </p>
*
* @param sessionHandle
* the session handle
@@ -389,18 +397,20 @@ public class LensMLImpl implements LensML {
* the model id
* @param queryRunner
* the query runner
+ * @param outputTable
+ * table where test output will be written
* @return the ML test report
* @throws LensException
* the lens exception
*/
public MLTestReport testModel(LensSessionHandle sessionHandle, String table, String algorithm, String modelID,
- TestQueryRunner queryRunner) throws LensException {
+ TestQueryRunner queryRunner, String outputTable) throws LensException {
// check if algorithm exists
if (!getAlgorithms().contains(algorithm)) {
throw new LensException("No such algorithm " + algorithm);
}
- MLModel model;
+ MLModel<?> model;
try {
model = ModelLoader.loadModel(conf, algorithm, modelID);
} catch (IOException e) {
@@ -418,22 +428,31 @@ public class LensMLImpl implements LensML {
}
String testID = UUID.randomUUID().toString().replace("-", "_");
- final String testTable = "ml_test_" + testID;
+ final String testTable = outputTable;
final String testResultColumn = "prediction_result";
// TODO support error metric UDAFs
TableTestingSpec spec = TableTestingSpec.newBuilder().hiveConf(conf)
- .database(database == null ? "default" : database).table(table).featureColumns(model.getFeatureColumns())
+ .database(database == null ? "default" : database).inputTable(table).featureColumns(model.getFeatureColumns())
.outputColumn(testResultColumn).labeColumn(model.getLabelColumn()).algorithm(algorithm).modelID(modelID)
- .outputTable(testTable).build();
- String testQuery = spec.getTestQuery();
+ .outputTable(testTable).testID(testID).build();
+ String testQuery = spec.getTestQuery();
if (testQuery == null) {
throw new LensException("Invalid test spec. " + "table=" + table + " algorithm=" + algorithm + " modelID="
+ modelID);
}
- LOG.info("Running test query " + testQuery);
+ if (!spec.isOutputTableExists()) {
+ LOG.info("Output table '" + testTable + "' does not exist for test algorithm = " + algorithm + " modelid="
+ + modelID + ", Creating table using query: " + spec.getCreateOutputTableQuery());
+ // create the output table
+ String createOutputTableQuery = spec.getCreateOutputTableQuery();
+ queryRunner.runQuery(createOutputTableQuery);
+ LOG.info("Table created " + testTable);
+ }
+
+ LOG.info("Running evaluation query " + testQuery);
QueryHandle testQueryHandle = queryRunner.runQuery(testQuery);
MLTestReport testReport = new MLTestReport();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/MLModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLModel.java
index 1b4b0ad..a9ba0fa 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLModel.java
@@ -21,6 +21,7 @@ package org.apache.lens.ml;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
+import lombok.ToString;
import java.io.Serializable;
import java.util.Date;
@@ -30,6 +31,7 @@ import java.util.List;
* Instantiates a new ML model.
*/
@NoArgsConstructor
+@ToString
public abstract class MLModel<PREDICTION> implements Serializable {
/** The id. */
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/MLTestReport.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLTestReport.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLTestReport.java
index 40b041b..0c70a8c 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLTestReport.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLTestReport.java
@@ -21,6 +21,7 @@ package org.apache.lens.ml;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
+import lombok.ToString;
import java.io.Serializable;
import java.util.List;
@@ -29,6 +30,7 @@ import java.util.List;
* Instantiates a new ML test report.
*/
@NoArgsConstructor
+@ToString
public class MLTestReport implements Serializable {
/** The test table. */
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/MLUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLUtils.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLUtils.java
index aed79d0..ce21d24 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/MLUtils.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/MLUtils.java
@@ -18,7 +18,23 @@
*/
package org.apache.lens.ml;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.ServiceProvider;
+import org.apache.lens.server.api.ServiceProviderFactory;
+import org.apache.lens.server.ml.MLService;
+import org.apache.lens.server.ml.MLServiceImpl;
+
public class MLUtils {
+
+ private static final HiveConf hiveConf;
+ static {
+ hiveConf = new HiveConf();
+ // Add default config so that we know the service provider implementation
+ hiveConf.addResource("lensserver-default.xml");
+ hiveConf.addResource("lens-site.xml");
+ }
+
public static String getTrainerName(Class<? extends MLTrainer> trainerClass) {
Algorithm annotation = trainerClass.getAnnotation(Algorithm.class);
if (annotation != null) {
@@ -26,4 +42,15 @@ public class MLUtils {
}
throw new IllegalArgumentException("Trainer should be decorated with annotation - " + Algorithm.class.getName());
}
+
+ public static MLServiceImpl getMLService() throws Exception {
+ return getServiceProvider().getService(MLService.NAME);
+ }
+
+ public static ServiceProvider getServiceProvider() throws Exception {
+ Class<? extends ServiceProviderFactory> spfClass = hiveConf.getClass(LensConfConstants.SERVICE_PROVIDER_FACTORY,
+ null, ServiceProviderFactory.class);
+ ServiceProviderFactory spf = spfClass.newInstance();
+ return spf.getServiceProvider();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/TableTestingSpec.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/TableTestingSpec.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/TableTestingSpec.java
index d4feeec..e73e989 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/TableTestingSpec.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/TableTestingSpec.java
@@ -18,6 +18,12 @@
*/
package org.apache.lens.ml;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import lombok.Getter;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -27,9 +33,6 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* Table specification for running test on a table.
*/
@@ -41,8 +44,9 @@ public class TableTestingSpec {
/** The db. */
private String db;
- /** The table. */
- private String table;
+ /** The table containing input data. */
+ private String inputTable;
+
// TODO use partition condition
/** The partition filter. */
private String partitionFilter;
@@ -68,6 +72,13 @@ public class TableTestingSpec {
/** The model id. */
private String modelID;
+ @Getter
+ private boolean outputTableExists;
+
+ public String testID;
+
+ private HashMap<String, FieldSchema> columnNameToFieldSchema;
+
/**
* The Class TableTestingSpecBuilder.
*/
@@ -96,19 +107,19 @@ public class TableTestingSpec {
}
/**
- * Table.
+ * Set the input table
*
* @param table
* the table
* @return the table testing spec builder
*/
- public TableTestingSpecBuilder table(String table) {
- spec.table = table;
+ public TableTestingSpecBuilder inputTable(String table) {
+ spec.inputTable = table;
return this;
}
/**
- * Partition filter.
+ * Partition filter for input table
*
* @param partFilter
* the part filter
@@ -211,6 +222,17 @@ public class TableTestingSpec {
public TableTestingSpec build() {
return spec;
}
+
+ /**
+ * Set the unique test id
+ *
+ * @param testID
+ * @return
+ */
+ public TableTestingSpecBuilder testID(String testID) {
+ spec.testID = testID;
+ return this;
+ }
}
/**
@@ -231,8 +253,17 @@ public class TableTestingSpec {
List<FieldSchema> columns;
try {
Hive metastoreClient = Hive.get(conf);
- Table tbl = (db == null) ? metastoreClient.getTable(table) : metastoreClient.getTable(db, table);
+ Table tbl = (db == null) ? metastoreClient.getTable(inputTable) : metastoreClient.getTable(db, inputTable);
columns = tbl.getAllCols();
+ columnNameToFieldSchema = new HashMap<String, FieldSchema>();
+
+ for (FieldSchema fieldSchema : columns) {
+ columnNameToFieldSchema.put(fieldSchema.getName(), fieldSchema);
+ }
+
+ // Check if output table exists
+ Table outTbl = metastoreClient.getTable(db == null ? "default" : db, outputTable, false);
+ outputTableExists = (outTbl != null);
} catch (HiveException exc) {
LOG.error("Error getting table info " + toString(), exc);
return false;
@@ -271,12 +302,33 @@ public class TableTestingSpec {
return null;
}
- StringBuilder q = new StringBuilder("CREATE TABLE " + outputTable + " AS SELECT ");
+ // We always insert a dynamic partition
+ StringBuilder q = new StringBuilder("INSERT OVERWRITE TABLE " + outputTable + " PARTITION (part_testid='" + testID
+ + "') SELECT ");
String featureCols = StringUtils.join(featureColumns, ",");
q.append(featureCols).append(",").append(labelColumn).append(", ").append("predict(").append("'").append(algorithm)
.append("', ").append("'").append(modelID).append("', ").append(featureCols).append(") ").append(outputColumn)
- .append(" FROM ").append(table);
+ .append(" FROM ").append(inputTable);
return q.toString();
}
+
+ public String getCreateOutputTableQuery() {
+ StringBuilder createTableQuery = new StringBuilder("CREATE TABLE IF NOT EXISTS ").append(outputTable).append("(");
+ // Output table contains feature columns, label column, output column
+ List<String> outputTableColumns = new ArrayList<String>();
+ for (String featureCol : featureColumns) {
+ outputTableColumns.add(featureCol + " " + columnNameToFieldSchema.get(featureCol).getType());
+ }
+
+ outputTableColumns.add(labelColumn + " " + columnNameToFieldSchema.get(labelColumn).getType());
+ outputTableColumns.add(outputColumn + " string");
+
+ createTableQuery.append(StringUtils.join(outputTableColumns, ", "));
+
+ // Append partition column
+ createTableQuery.append(") PARTITIONED BY (part_testid string)");
+
+ return createTableQuery.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MLDBUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MLDBUtils.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MLDBUtils.java
new file mode 100644
index 0000000..4aecae4
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MLDBUtils.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.dao;
+
+import org.apache.lens.ml.MLModel;
+import org.apache.lens.ml.MLTestReport;
+import org.apache.lens.ml.task.MLTask;
+
+public class MLDBUtils {
+
+ /**
+ * Create table to store test report data
+ */
+ public void createTestReportTable() {
+
+ }
+
+ /**
+ * Create table to store ML task workflow data
+ */
+ public void createMLTaskTable() {
+
+ }
+
+ /**
+ * Create table to save ML Models
+ */
+ public void createMLModelTable() {
+
+ }
+
+ /**
+ * Insert an ML Task into ml task table
+ *
+ * @param task
+ */
+ public void saveMLTask(MLTask task) {
+
+ }
+
+ /**
+ * Get ML Task given its id
+ *
+ * @param taskID
+ * @return
+ */
+ public MLTask getMLTask(String taskID) {
+ return null;
+ }
+
+ /**
+ * Insert test report into test report table
+ *
+ * @param testReport
+ */
+ public void saveTestReport(MLTestReport testReport) {
+
+ }
+
+ /**
+ * Get test report given its ID
+ *
+ * @param testReportID
+ * @return
+ */
+ public MLTestReport getTestReport(String testReportID) {
+ return null;
+ }
+
+ /**
+ * Insert model metadata into model table
+ *
+ * @param mlModel
+ */
+ public void saveMLModel(MLModel<?> mlModel) {
+
+ }
+
+ /**
+ * Get model metadata given ID
+ *
+ * @param modelID
+ * @return
+ */
+ public MLModel<?> getMLModel(String modelID) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/ColumnFeatureFunction.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/ColumnFeatureFunction.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/ColumnFeatureFunction.java
index 1a75af6..0a9fa09 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/ColumnFeatureFunction.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/ColumnFeatureFunction.java
@@ -55,7 +55,7 @@ public class ColumnFeatureFunction extends FeatureFunction {
/**
* Feature positions and value mappers are parallel arrays. featurePositions[i] gives the position of ith feature in
* the HCatRecord, and valueMappers[i] gives the value mapper used to map that feature to a Double value
- *
+ *
* @param featurePositions
* position number of feature column in the HCatRecord
* @param valueMappers
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/DoubleValueMapper.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/DoubleValueMapper.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/DoubleValueMapper.java
index 5798a10..f6a2c0b 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/DoubleValueMapper.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/DoubleValueMapper.java
@@ -32,6 +32,10 @@ public class DoubleValueMapper extends FeatureValueMapper {
*/
@Override
public final Double call(Object input) {
- return input == null ? 0d : (Double) input;
+ if (input instanceof Double || input == null) {
+ return input == null ? Double.valueOf(0d) : (Double) input;
+ }
+
+ throw new IllegalArgumentException("Invalid input expecting only doubles, but got " + input);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/HiveTableRDD.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/HiveTableRDD.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/HiveTableRDD.java
index d819258..fc4456d 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/HiveTableRDD.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/HiveTableRDD.java
@@ -18,9 +18,13 @@
*/
package org.apache.lens.ml.spark;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hive.hcatalog.data.HCatRecord;
+import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
@@ -31,6 +35,7 @@ import java.io.IOException;
* Create a JavaRDD based on a Hive table using HCatInputFormat.
*/
public class HiveTableRDD {
+ public static final Log LOG = LogFactory.getLog(HiveTableRDD.class);
/**
* Creates the hive table rdd.
@@ -53,12 +58,10 @@ public class HiveTableRDD {
Configuration conf, String db, String table, String partitionFilter) throws IOException {
HCatInputFormat.setInput(conf, db, table, partitionFilter);
+
JavaPairRDD<WritableComparable, HCatRecord> rdd = javaSparkContext.newAPIHadoopRDD(conf, HCatInputFormat.class, // Input
- // format
- // class
WritableComparable.class, // input key class
HCatRecord.class); // input value class
-
return rdd;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/TableTrainingSpec.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/TableTrainingSpec.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/TableTrainingSpec.java
index d301961..7be4c08 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/TableTrainingSpec.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/TableTrainingSpec.java
@@ -18,19 +18,23 @@
*/
package org.apache.lens.ml.spark;
-import com.google.common.base.Preconditions;
-import org.apache.lens.api.LensException;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
import lombok.Getter;
-import org.apache.commons.lang.StringUtils;
+import lombok.ToString;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hive.hcatalog.data.HCatRecord;
+import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.lens.api.LensException;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
@@ -38,14 +42,12 @@ import org.apache.spark.api.java.function.Function;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.rdd.RDD;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
+import com.google.common.base.Preconditions;
/**
* The Class TableTrainingSpec.
*/
+@ToString
public class TableTrainingSpec implements Serializable {
/** The Constant LOG. */
@@ -326,21 +328,23 @@ public class TableTrainingSpec implements Serializable {
* @return true, if successful
*/
boolean validate() {
- List<FieldSchema> columns;
+ List<HCatFieldSchema> columns;
try {
- Hive metastoreClient = Hive.get(conf);
- Table tbl = (database == null) ? metastoreClient.getTable(table) : metastoreClient.getTable(database, table);
- columns = tbl.getAllCols();
- } catch (HiveException exc) {
+ HCatInputFormat.setInput(conf, database == null ? "default" : database, table, partitionFilter);
+ HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ columns = tableSchema.getFields();
+ } catch (IOException exc) {
LOG.error("Error getting table info " + toString(), exc);
return false;
}
+ LOG.info(table + " columns " + columns.toString());
+
boolean valid = false;
if (columns != null && !columns.isEmpty()) {
// Check labeled column
List<String> columnNames = new ArrayList<String>();
- for (FieldSchema col : columns) {
+ for (HCatFieldSchema col : columns) {
columnNames.add(col.getName());
}
@@ -396,6 +400,8 @@ public class TableTrainingSpec implements Serializable {
throw new LensException("Table spec not valid: " + toString());
}
+ LOG.info("Creating RDDs with spec " + toString());
+
// Get the RDD for table
JavaPairRDD<WritableComparable, HCatRecord> tableRDD;
try {
@@ -435,13 +441,4 @@ public class TableTrainingSpec implements Serializable {
LOG.info("Generated RDDs");
}
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#toString()
- */
- @Override
- public String toString() {
- return StringUtils.join(new String[] { database, table, partitionFilter, labelColumn }, ",");
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/trainers/BaseSparkTrainer.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/trainers/BaseSparkTrainer.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/trainers/BaseSparkTrainer.java
index 92377eb..115fef3 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/trainers/BaseSparkTrainer.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/spark/trainers/BaseSparkTrainer.java
@@ -117,9 +117,10 @@ public abstract class BaseSparkTrainer implements MLTrainer {
* java.lang.String, java.lang.String[])
*/
@Override
- public MLModel train(LensConf conf, String db, String table, String modelId, String... params) throws LensException {
+ public MLModel<?> train(LensConf conf, String db, String table, String modelId, String... params)
+ throws LensException {
parseParams(params);
- LOG.info("Training " + " with " + features.size() + " features");
+
TableTrainingSpec.TableTrainingSpecBuilder builder = TableTrainingSpec.newBuilder().hiveConf(toHiveConf(conf))
.database(db).table(table).partitionFilter(partitionFilter).featureColumns(features).labelColumn(label);
@@ -128,10 +129,12 @@ public abstract class BaseSparkTrainer implements MLTrainer {
}
TableTrainingSpec spec = builder.build();
+ LOG.info("Training " + " with " + features.size() + " features");
+
spec.createRDDs(sparkContext);
RDD<LabeledPoint> trainingRDD = spec.getTrainingRDD();
- BaseSparkClassificationModel model = trainInternal(modelId, trainingRDD);
+ BaseSparkClassificationModel<?> model = trainInternal(modelId, trainingRDD);
model.setTable(table);
model.setParams(Arrays.asList(params));
model.setLabelColumn(label);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
new file mode 100644
index 0000000..85fd01d
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/task/MLTask.java
@@ -0,0 +1,306 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.task;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import lombok.Getter;
+import lombok.ToString;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.client.LensConnectionParams;
+import org.apache.lens.client.LensMLClient;
+import org.apache.lens.ml.LensML;
+import org.apache.lens.ml.MLTestReport;
+import org.apache.lens.ml.MLUtils;
+
+/**
+ * Run a complete cycle of train and test (evaluation) for an ML algorithm
+ */
+@ToString
+public class MLTask implements Runnable {
+ private static final Log LOG = LogFactory.getLog(MLTask.class);
+
+ public enum State {
+ RUNNING, SUCCESSFUL, FAILED
+ }
+
+ @Getter
+ private State taskState;
+
+ /**
+ * Name of the trainer/algorithm.
+ */
+ @Getter
+ private String algorithm;
+
+ /**
+ * Name of the table containing training data.
+ */
+ @Getter
+ private String trainingTable;
+
+ /**
+ * Training table partition spec
+ */
+ @Getter
+ private String partitionSpec;
+
+ /**
+ * Name of the column which is a label for supervised algorithms.
+ */
+ @Getter
+ private String labelColumn;
+
+ /**
+ * Names of columns which are features in the training data.
+ */
+ @Getter
+ private List<String> featureColumns;
+
+ /**
+ * Configuration for the example.
+ */
+ @Getter
+ private HiveConf configuration;
+
+ /**
+ * Lens Server base URL, when running example as a client.
+ */
+ @Getter
+ private String serverLocation;
+
+ private LensML ml;
+ private String taskID;
+
+ /**
+ * Output table name
+ */
+ @Getter
+ private String outputTable;
+
+ /**
+ * Session handle
+ */
+ @Getter
+ private LensSessionHandle sessionHandle;
+
+ /**
+ * Extra params passed to the training algorithm
+ */
+ @Getter
+ private Map<String, String> extraParams;
+
+ /**
+ * User name to connect to Lens server
+ */
+ @Getter
+ public String userName;
+
+ /**
+ * Password to connect to Lens server
+ */
+ @Getter
+ public String password;
+
+ /**
+ * Use ExampleTask.Builder to create an instance
+ */
+ private MLTask() {
+ // Use builder to construct the example
+ extraParams = new HashMap<String, String>();
+ taskID = UUID.randomUUID().toString();
+ }
+
+ /**
+ * Builder to create an example task
+ */
+ public static class Builder {
+ private MLTask task;
+
+ public Builder() {
+ task = new MLTask();
+ }
+
+ public Builder trainingTable(String trainingTable) {
+ task.trainingTable = trainingTable;
+ return this;
+ }
+
+ public Builder algorithm(String algorithm) {
+ task.algorithm = algorithm;
+ return this;
+ }
+
+ public Builder labelColumn(String labelColumn) {
+ task.labelColumn = labelColumn;
+ return this;
+ }
+
+ public Builder addFeatureColumn(String featureColumn) {
+ if (task.featureColumns == null) {
+ task.featureColumns = new ArrayList<String>();
+ }
+ task.featureColumns.add(featureColumn);
+ return this;
+ }
+
+ public Builder hiveConf(HiveConf hiveConf) {
+ task.configuration = hiveConf;
+ return this;
+ }
+
+ public Builder serverLocation(String serverLocation) {
+ task.serverLocation = serverLocation;
+ return this;
+ }
+
+ public Builder sessionHandle(LensSessionHandle sessionHandle) {
+ task.sessionHandle = sessionHandle;
+ return this;
+ }
+
+ public Builder extraParam(String param, String value) {
+ task.extraParams.put(param, value);
+ return this;
+ }
+
+ public Builder partitionSpec(String partitionSpec) {
+ task.partitionSpec = partitionSpec;
+ return this;
+ }
+
+ public Builder outputTable(String outputTable) {
+ task.outputTable = outputTable;
+ return this;
+ }
+
+ public MLTask build() {
+ MLTask builtTask = task;
+ task = null;
+ return builtTask;
+ }
+
+ public Builder userName(String userName) {
+ task.userName = userName;
+ return this;
+ }
+
+ public Builder password(String password) {
+ task.password = password;
+ return this;
+ }
+ }
+
+ @Override
+ public void run() {
+ taskState = State.RUNNING;
+ LOG.info("Starting " + taskID);
+ try {
+ runTask();
+ taskState = State.SUCCESSFUL;
+ LOG.info("Complete " + taskID);
+ } catch (Exception e) {
+ taskState = State.FAILED;
+ LOG.info("Error running task " + taskID, e);
+ }
+ }
+
+ /**
+ * Train an ML model, with specified algorithm and input data. Do model evaluation using the evaluation data and print
+ * evaluation result
+ *
+ * @throws Exception
+ */
+ private void runTask() throws Exception {
+ if (serverLocation != null) {
+ // Connect to a remote Lens server
+ LensConnectionParams connectionParams = new LensConnectionParams();
+ connectionParams.setBaseUrl(serverLocation);
+ connectionParams.getConf().setUser(userName);
+ LensMLClient mlClient = new LensMLClient(connectionParams, sessionHandle);
+ ml = mlClient;
+ LOG.info("Working in client mode. Lens session handle " + sessionHandle.getPublicId());
+ } else {
+ // In server mode session handle has to be passed by the user as a request parameter
+ ml = MLUtils.getMLService();
+ LOG.info("Working in Lens server");
+ }
+
+ String trainerArgs[] = buildTrainingArgs();
+ LOG.info("Starting task " + taskID + " trainer args: " + Arrays.toString(trainerArgs));
+
+ String modelID = ml.train(trainingTable, algorithm, trainerArgs);
+ printModelMetadata(taskID, modelID);
+
+ LOG.info("Starting test " + taskID);
+ MLTestReport testReport = ml.testModel(sessionHandle, trainingTable, algorithm, modelID, outputTable);
+ printTestReport(taskID, testReport);
+ saveTask();
+ }
+
+ // Save task metadata to DB
+ private void saveTask() {
+ LOG.info("Saving task details to DB");
+ }
+
+ private void printTestReport(String exampleID, MLTestReport testReport) {
+ StringBuilder builder = new StringBuilder("Example: ").append(exampleID);
+ builder.append("\n\t");
+ builder.append("EvaluationReport: ").append(testReport.toString());
+ System.out.println(builder.toString());
+ }
+
+ private String[] buildTrainingArgs() {
+ List<String> argList = new ArrayList<String>();
+ argList.add("label");
+ argList.add(labelColumn);
+
+ // Add all the features
+ for (String featureCol : featureColumns) {
+ argList.add("feature");
+ argList.add(featureCol);
+ }
+
+ // Add extra params
+ for (String param : extraParams.keySet()) {
+ argList.add(param);
+ argList.add(extraParams.get(param));
+ }
+
+ return argList.toArray(new String[argList.size()]);
+ }
+
+ // Get the model instance and print its metadat to stdout
+ private void printModelMetadata(String exampleID, String modelID) throws Exception {
+ StringBuilder builder = new StringBuilder("Example: ").append(exampleID);
+ builder.append("\n\t");
+ builder.append("Model: ");
+ builder.append(ml.getModel(algorithm, modelID).toString());
+ System.out.println(builder.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLApp.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLApp.java b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLApp.java
index 4388d23..75d4f03 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLApp.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLApp.java
@@ -29,13 +29,32 @@ import org.glassfish.jersey.media.multipart.MultiPartFeature;
@ApplicationPath("/ml")
public class MLApp extends Application {
- @Override
- public Set<Class<?>> getClasses() {
- final Set<Class<?>> classes = new HashSet<Class<?>>();
+
+ private final Set<Class<?>> classes;
+
+ /**
+ * Pass additional classes when running in test mode
+ *
+ * @param additionalClasses
+ */
+ public MLApp(Class<?>... additionalClasses) {
+ classes = new HashSet<Class<?>>();
+
// register root resource
classes.add(MLServiceResource.class);
classes.add(MultiPartFeature.class);
classes.add(LoggingFilter.class);
+ for (Class<?> cls : additionalClasses) {
+ classes.add(cls);
+ }
+
+ }
+
+ /**
+ * Get classes for this resource
+ */
+ @Override
+ public Set<Class<?>> getClasses() {
return classes;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceImpl.java
index d1b5ff3..be7cd27 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceImpl.java
@@ -32,6 +32,7 @@ import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hive.service.CompositeService;
import org.apache.lens.server.ml.MLService;
@@ -203,10 +204,10 @@ public class MLServiceImpl extends CompositeService implements MLService {
* java.lang.String)
*/
@Override
- public MLTestReport testModel(LensSessionHandle sessionHandle, String table, String algorithm, String modelID)
- throws LensException {
+ public MLTestReport testModel(LensSessionHandle sessionHandle, String table, String algorithm, String modelID,
+ String outputTable) throws LensException {
- return ml.testModel(sessionHandle, table, algorithm, modelID, new DirectQueryRunner(sessionHandle));
+ return ml.testModel(sessionHandle, table, algorithm, modelID, new DirectQueryRunner(sessionHandle), outputTable);
}
/*
@@ -281,6 +282,8 @@ public class MLServiceImpl extends CompositeService implements MLService {
*/
@Override
public QueryHandle runQuery(String testQuery) throws LensException {
+ FunctionRegistry.registerTemporaryFunction("predict", HiveMLUDF.class);
+ LOG.info("Registered predict UDF");
// Run the query in query executions service
QueryExecutionService queryService = (QueryExecutionService) getServiceProvider().getService("query");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceResource.java b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceResource.java
index 2e9bbaf..08fc64b 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceResource.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/server/ml/MLServiceResource.java
@@ -37,6 +37,7 @@ import org.glassfish.jersey.media.multipart.FormDataParam;
import javax.ws.rs.*;
import javax.ws.rs.core.*;
+
import java.util.*;
import static org.apache.commons.lang.StringUtils.isBlank;
@@ -62,6 +63,11 @@ public class MLServiceResource {
private static final HiveConf hiveConf;
+ /**
+ * Message indicating if ML service is up
+ */
+ public static final String ML_UP_MESSAGE = "ML service is up";
+
static {
hiveConf = new HiveConf();
// Add default config so that we know the service provider implementation
@@ -109,6 +115,16 @@ public class MLServiceResource {
}
/**
+ * Indicates if ML resource is up
+ *
+ * @return
+ */
+ @GET
+ public String mlResourceUp() {
+ return ML_UP_MESSAGE;
+ }
+
+ /**
* Get a list of trainers available
*
* @return
@@ -122,7 +138,7 @@ public class MLServiceResource {
}
/**
- * Gets the param description.
+ * Gets the human readable param description of an algorithm
*
* @param algorithm
* the algorithm
@@ -277,10 +293,9 @@ public class MLServiceResource {
trainerArgs.add(values.get(0));
}
}
-
+ LOG.info("Training table " + table + " with algo " + algorithm + " params=" + trainerArgs.toString());
String modelId = getMlService().train(table, algorithm, trainerArgs.toArray(new String[] {}));
- LOG.info("Trained table " + table + " with algo " + algorithm + " params=" + trainerArgs.toString() + ", modelID="
- + modelId);
+ LOG.info("Done training " + table + " modelid = " + modelId);
return modelId;
}
@@ -317,8 +332,9 @@ public class MLServiceResource {
@Path("test/{table}/{algorithm}/{modelID}")
@Consumes(MediaType.MULTIPART_FORM_DATA)
public String test(@PathParam("algorithm") String algorithm, @PathParam("modelID") String modelID,
- @PathParam("table") String table, @FormDataParam("sessionid") LensSessionHandle session) throws LensException {
- MLTestReport testReport = getMlService().testModel(session, table, algorithm, modelID);
+ @PathParam("table") String table, @FormDataParam("sessionid") LensSessionHandle session,
+ @FormDataParam("outputTable") String outputTable) throws LensException {
+ MLTestReport testReport = getMlService().testModel(session, table, algorithm, modelID, outputTable);
return testReport.getReportID();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
index 3fe0225..4abc010 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
@@ -18,21 +18,27 @@
*/
package org.apache.lens.ml;
+import java.io.File;
+import java.net.URI;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Application;
-import javax.ws.rs.core.MediaType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.Service;
-import org.apache.lens.api.StringList;
+import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.client.LensConnectionParams;
+import org.apache.lens.client.LensMLClient;
import org.apache.lens.ml.spark.trainers.DecisionTreeTrainer;
import org.apache.lens.ml.spark.trainers.LogisticRegressionTrainer;
import org.apache.lens.ml.spark.trainers.NaiveBayesTrainer;
import org.apache.lens.ml.spark.trainers.SVMTrainer;
+import org.apache.lens.ml.task.MLTask;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.LensServerConf;
import org.apache.lens.server.api.LensConfConstants;
@@ -41,8 +47,13 @@ import org.apache.lens.server.api.ServiceProviderFactory;
import org.apache.lens.server.ml.MLApp;
import org.apache.lens.server.ml.MLService;
import org.apache.lens.server.ml.MLServiceImpl;
+import org.apache.lens.server.ml.MLServiceResource;
+import org.apache.lens.server.query.QueryServiceResource;
+import org.apache.lens.server.session.HiveSessionService;
+import org.apache.lens.server.session.SessionResource;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import org.glassfish.jersey.server.ResourceConfig;
import org.testng.Assert;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeMethod;
@@ -52,14 +63,25 @@ import org.testng.annotations.Test;
@Test
public class TestMLResource extends LensJerseyTest {
private static final Log LOG = LogFactory.getLog(TestMLResource.class);
+ private static final String TEST_CONN_URL = "http://localhost:8089/lens-server";
+ private static final LensConnectionParams connectionParams = new LensConnectionParams();
+
+ static {
+ connectionParams.setBaseUrl(TEST_CONN_URL);
+ connectionParams.getConf().setUser("foo@localhost");
+ }
+
private WebTarget mlTarget;
+ private LensMLClient mlClient;
+ private ServiceProvider serviceProvider;
+ private LensSessionHandle sessionHandle;
- public static ServiceProvider getServiceProvider() throws Exception {
+ public void setServiceProvider() throws Exception {
HiveConf conf = LensServerConf.get();
Class<? extends ServiceProviderFactory> spfClass = conf.getClass(LensConfConstants.SERVICE_PROVIDER_FACTORY, null,
ServiceProviderFactory.class);
ServiceProviderFactory spf = spfClass.newInstance();
- return spf.getServiceProvider();
+ this.serviceProvider = spf.getServiceProvider();
}
@Override
@@ -69,7 +91,7 @@ public class TestMLResource extends LensJerseyTest {
@Override
protected Application configure() {
- return new MLApp();
+ return new MLApp(SessionResource.class, QueryServiceResource.class);
}
@Override
@@ -80,35 +102,39 @@ public class TestMLResource extends LensJerseyTest {
@BeforeTest
public void setUp() throws Exception {
super.setUp();
+ setServiceProvider();
+ HiveSessionService sessionService = serviceProvider.getService(HiveSessionService.NAME);
+ this.sessionHandle = sessionService.openSession("foo@localhost", "bar", new HashMap<String, String>());
+ mlClient = new LensMLClient(connectionParams, sessionHandle);
}
@AfterTest
public void tearDown() throws Exception {
super.tearDown();
+ mlClient.close();
}
@BeforeMethod
public void setMLTarget() {
mlTarget = target().path("ml");
- LOG.info("## setMLTarget");
}
@Test
public void testStartMLServiceStarted() throws Exception {
LOG.info("## testStartMLServiceStarted");
- ServiceProvider serviceProvider = getServiceProvider();
MLServiceImpl svcImpl = serviceProvider.getService(MLService.NAME);
Assert.assertEquals(svcImpl.getServiceState(), Service.STATE.STARTED);
}
@Test
- public void testGetTrainers() throws Exception {
- final WebTarget trainerTarget = mlTarget.path("trainers");
- LOG.info("## testGetTrainers: " + trainerTarget.getUri());
- StringList trainerList = trainerTarget.request(MediaType.APPLICATION_XML).get(StringList.class);
- Assert.assertNotNull(trainerList);
+ public void testMLResourceUp() throws Exception {
+ String mlUpMsg = mlTarget.request().get(String.class);
+ Assert.assertEquals(mlUpMsg, MLServiceResource.ML_UP_MESSAGE);
+ }
- List<String> trainerNames = trainerList.getElements();
+ @Test
+ public void testGetTrainers() throws Exception {
+ List<String> trainerNames = mlClient.getAlgorithms();
Assert.assertNotNull(trainerNames);
Assert.assertTrue(trainerNames.contains(MLUtils.getTrainerName(NaiveBayesTrainer.class)),
@@ -124,4 +150,55 @@ public class TestMLResource extends LensJerseyTest {
MLUtils.getTrainerName(DecisionTreeTrainer.class));
}
+ @Test
+ public void testGetTrainerParams() throws Exception {
+ Map<String, String> params = mlClient.getAlgoParamDescription(MLUtils.getTrainerName(DecisionTreeTrainer.class));
+ Assert.assertNotNull(params);
+ Assert.assertFalse(params.isEmpty());
+
+ for (String key : params.keySet()) {
+ LOG.info("## Param " + key + " help = " + params.get(key));
+ }
+ }
+
+ @Test
+ public void trainAndEval() throws Exception {
+ LOG.info("Starting train & eval");
+ final String algoName = MLUtils.getTrainerName(NaiveBayesTrainer.class);
+ HiveConf conf = new HiveConf();
+ String database = "default";
+ String tableName = "naivebayes_training_table";
+ String sampleDataFilePath = "data/naive_bayes/naive_bayes_train.data";
+
+ File sampleDataFile = new File(sampleDataFilePath);
+ URI sampleDataFileURI = sampleDataFile.toURI();
+
+ String labelColumn = "label";
+ String features[] = { "feature_1", "feature_2", "feature_3" };
+ String outputTable = "naivebayes_eval_table";
+
+ LOG.info("Creating training table from file " + sampleDataFileURI.toString());
+
+ Map<String, String> tableParams = new HashMap<String, String>();
+ ExampleUtils.createTable(conf, database, tableName, sampleDataFileURI.toString(), labelColumn, tableParams,
+ features);
+ MLTask.Builder taskBuilder = new MLTask.Builder();
+
+ taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn).outputTable(outputTable)
+ .serverLocation(getBaseUri().toString()).sessionHandle(mlClient.getSessionHandle()).trainingTable(tableName)
+ .userName("foo@localhost").password("bar");
+
+ // Add features
+ taskBuilder.addFeatureColumn("feature_1").addFeatureColumn("feature_2").addFeatureColumn("feature_3");
+
+ MLTask task = taskBuilder.build();
+
+ LOG.info("Created task " + task.toString());
+ task.run();
+ Assert.assertEquals(task.getTaskState(), MLTask.State.SUCCESSFUL);
+
+ LOG.info("Completed task run");
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-ml-lib/testutils/hadoop
----------------------------------------------------------------------
diff --git a/lens-ml-lib/testutils/hadoop b/lens-ml-lib/testutils/hadoop
new file mode 100755
index 0000000..415561f
--- /dev/null
+++ b/lens-ml-lib/testutils/hadoop
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script satisfies Hive's bin/hadoop dependency when running tests.
+# It's a quick and dirty hack, and should eventually be made obsolete by
+# completely eliminating Hive's dependency on bin/hadoop.
+
+bin=`which $0`
+bin=`dirname ${bin}`
+bin=`cd "$bin"; pwd`
+
+# Newer versions of glibc use an arena memory allocator that causes virtual
+# memory usage to explode. This interacts badly with the many threads that
+# we use in Hadoop. Tune the variable down to prevent vmem explosion.
+export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m
+
+# check envvars which might override default args
+if [ "$HADOOP_HEAPSIZE" != "" ]; then
+ JAVA_HEAP_MAX="-Xmx""$HADOOP_HEAPSIZE""m"
+fi
+
+# CLASSPATH initially contains $HADOOP_CONF_DIR
+CLASSPATH="${HADOOP_CONF_DIR}"
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+BUILD_ROOT=./target/hadoop_logs
+
+#if [ "$HIVE_HADOOP_TEST_CLASSPATH" != "" ]; then
+# CLASSPATH=${CLASSPATH}:${HIVE_HADOOP_TEST_CLASSPATH}
+#else
+# echo "Error: HIVE_HADOOP_TEST_CLASSPATH not defined."
+# exit 1
+#fi
+
+# restore ordinary behaviour
+unset IFS
+
+mkdir -p $BUILD_ROOT/test/hadoop/logs
+HADOOP_LOG_DIR=$BUILD_ROOT/test/hadoop/logs
+HADOOP_LOGFILE=hadoop.log
+
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+ HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.policy.file=$HADOOP_POLICYFILE"
+
+# Disable ipv6 as it can cause issues
+HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
+
+COMMAND=$1
+# the core commands
+if [ "$COMMAND" = "fs" ] ; then
+ CLASS=org.apache.hadoop.fs.FsShell
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "version" ] ; then
+ CLASS=org.apache.hadoop.util.VersionInfo
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "jar" ] ; then
+ CLASS=org.apache.hadoop.util.RunJar
+elif [ "$COMMAND" = "distcp" ] ; then
+ CLASS=org.apache.hadoop.tools.DistCp
+ CLASSPATH=${CLASSPATH}:${TOOL_PATH}
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "daemonlog" ] ; then
+ CLASS=org.apache.hadoop.log.LogLevel
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "archive" ] ; then
+ CLASS=org.apache.hadoop.tools.HadoopArchives
+ CLASSPATH=${CLASSPATH}:${TOOL_PATH}
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [[ "$COMMAND" = -* ]] ; then
+ # class and package names cannot begin with a -
+ echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'"
+ exit 1
+else
+ HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+ CLASS=$COMMAND
+fi
+shift
+
+#make sure security appender is turned off
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
+
+export CLASSPATH=$TEST_CLASSPATH:$CLASSPATH:`cat $MVN_CLASSPATH_FILE`
+echo "Classpath file -->" $MVN_CLASSPATH_FILE
+exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/88b900c3/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index e0a47bf..1dfb0b6 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -137,4 +137,16 @@
<description>Implementation class for session service</description>
</property>
+<property>
+ <name>lens.server.ws.resourcenames</name>
+ <value>session,metastore,query,quota,scheduler,index,ml</value>
+ <description>These JAX-RS resources would be started in the specified order when lens-server starts up</description>
+</property>
+
+<property>
+ <name>lens.server.ml.ws.resource.impl</name>
+ <value>org.apache.lens.server.ml.MLServiceResource</value>
+ <description>Implementation class for Session Resource</description>
+</property>
+
</configuration>
[17/50] incubator-lens git commit: Merge pull request #167 from
InMobi/cube-client-caching
Posted by am...@apache.org.
Merge pull request #167 from InMobi/cube-client-caching
Create one cube client for one dataabse
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/27cab4a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/27cab4a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/27cab4a6
Branch: refs/heads/toapache
Commit: 27cab4a6473b5a829ffee0dc12f70bda1711fe60
Parents: 75ce83f 59eb990
Author: Amareshwari <am...@inmobi.com>
Authored: Fri Nov 7 15:41:05 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Fri Nov 7 15:41:05 2014 +0530
----------------------------------------------------------------------
.../lens/cube/metadata/CubeMetastoreClient.java | 21 +++++++-------------
.../cube/metadata/TestCubeMetastoreClient.java | 5 ++---
.../apache/lens/cube/parse/CubeTestSetup.java | 4 ++--
.../metastore/CubeMetastoreServiceImpl.java | 3 ++-
.../lens/server/session/LensSessionImpl.java | 14 ++++---------
5 files changed, 17 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
[16/50] incubator-lens git commit: Create one cube client for one
dataabse
Posted by am...@apache.org.
Create one cube client for one dataabse
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/59eb990a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/59eb990a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/59eb990a
Branch: refs/heads/toapache
Commit: 59eb990a42b1b9f256fa23d89fadc498882a0068
Parents: 01a5c35
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Fri Nov 7 15:22:08 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Fri Nov 7 15:22:08 2014 +0530
----------------------------------------------------------------------
.../lens/cube/metadata/CubeMetastoreClient.java | 21 +++++++-------------
.../cube/metadata/TestCubeMetastoreClient.java | 5 ++---
.../apache/lens/cube/parse/CubeTestSetup.java | 4 ++--
.../metastore/CubeMetastoreServiceImpl.java | 3 ++-
.../lens/server/session/LensSessionImpl.java | 14 ++++---------
5 files changed, 17 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/59eb990a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 5eb2872..87c6fb4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -74,9 +74,9 @@ public class CubeMetastoreClient {
// map from storage name to storage
private final Map<String, Storage> allStorages = new HashMap<String, Storage>();
- // HiveConf to client mapping
- private static final Map<HiveConf, CubeMetastoreClient> clientMapping =
- new HashMap<HiveConf, CubeMetastoreClient>();
+ // dbname to client mapping
+ private static final Map<String, CubeMetastoreClient> clientMapping =
+ new HashMap<String, CubeMetastoreClient>();
private SchemaGraph schemaGraph;
@@ -89,10 +89,11 @@ public class CubeMetastoreClient {
* @throws HiveException
*/
public static CubeMetastoreClient getInstance(HiveConf conf) throws HiveException {
- if (clientMapping.get(conf) == null) {
- clientMapping.put(conf, new CubeMetastoreClient(conf));
+ String currentdb = SessionState.get().getCurrentDatabase();
+ if (clientMapping.get(currentdb) == null) {
+ clientMapping.put(currentdb, new CubeMetastoreClient(conf));
}
- return clientMapping.get(conf);
+ return clientMapping.get(currentdb);
}
private Hive getClient() throws HiveException {
@@ -106,14 +107,6 @@ public class CubeMetastoreClient {
Hive.closeCurrent();
}
- public void setCurrentDatabase(String currentDatabase) {
- SessionState.get().setCurrentDatabase(currentDatabase);
- }
-
- public String getCurrentDatabase() {
- return SessionState.get().getCurrentDatabase();
- }
-
private void createStorageHiveTable(Table parent, String storage, StorageTableDesc crtTblDesc) throws HiveException {
try {
Table tbl = getStorage(storage).getStorageTable(getClient(), parent, crtTblDesc);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/59eb990a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index ec17661..da54937 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -104,7 +104,6 @@ public class TestCubeMetastoreClient {
@BeforeClass
public static void setup() throws HiveException, AlreadyExistsException, ParseException {
SessionState.start(conf);
- client = CubeMetastoreClient.getInstance(conf);
now = new Date();
Calendar cal = Calendar.getInstance();
cal.add(Calendar.HOUR_OF_DAY, 1);
@@ -112,7 +111,8 @@ public class TestCubeMetastoreClient {
Database database = new Database();
database.setName(TestCubeMetastoreClient.class.getSimpleName());
Hive.get(conf).createDatabase(database);
- client.setCurrentDatabase(TestCubeMetastoreClient.class.getSimpleName());
+ SessionState.get().setCurrentDatabase(TestCubeMetastoreClient.class.getSimpleName());
+ client = CubeMetastoreClient.getInstance(conf);
defineCube(cubeName, cubeNameWithProps, derivedCubeName, derivedCubeNameWithProps);
defineUberDims();
}
@@ -395,7 +395,6 @@ public class TestCubeMetastoreClient {
@Test(priority = 1)
public void testCube() throws Exception {
- Assert.assertEquals(client.getCurrentDatabase(), this.getClass().getSimpleName());
client.createCube(cubeName, cubeMeasures, cubeDimensions, cubeExpressions, new HashMap<String, String>());
Assert.assertTrue(client.tableExists(cubeName));
Table cubeTbl = client.getHiveTable(cubeName);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/59eb990a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 5539787..e40db8d 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -1467,11 +1467,11 @@ public class CubeTestSetup {
public void createSources(HiveConf conf, String dbName) throws Exception {
try {
- CubeMetastoreClient client = CubeMetastoreClient.getInstance(conf);
Database database = new Database();
database.setName(dbName);
Hive.get(conf).createDatabase(database);
- client.setCurrentDatabase(dbName);
+ SessionState.get().setCurrentDatabase(dbName);
+ CubeMetastoreClient client = CubeMetastoreClient.getInstance(conf);
client.createStorage(new HDFSStorage(c1));
client.createStorage(new HDFSStorage(c2));
client.createStorage(new HDFSStorage(c3));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/59eb990a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index c79f4b8..23e0c7a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -677,7 +677,8 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
}
@Override
- public void addPartitionToFactStorage(LensSessionHandle sessionid, String fact, String storageName, XPartition partition) throws LensException {
+ public void addPartitionToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
+ XPartition partition) throws LensException {
try {
acquire(sessionid);
CubeFactTable factTable = checkFactStorage(sessionid, fact, storageName);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/59eb990a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
index 0ef5db4..32a43f7 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
@@ -49,9 +49,6 @@ public class LensSessionImpl extends HiveSessionImpl {
/** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(LensSessionImpl.class);
- /** The cube client. */
- private CubeMetastoreClient cubeClient;
-
/** The persist info. */
private LensSessionPersistInfo persistInfo = new LensSessionPersistInfo();
@@ -166,14 +163,11 @@ public class LensSessionImpl extends HiveSessionImpl {
}
public CubeMetastoreClient getCubeMetastoreClient() throws LensException {
- if (cubeClient == null) {
- try {
- cubeClient = CubeMetastoreClient.getInstance(getHiveConf());
- } catch (HiveException e) {
- throw new LensException(e);
- }
+ try {
+ return CubeMetastoreClient.getInstance(getHiveConf());
+ } catch (HiveException e) {
+ throw new LensException(e);
}
- return cubeClient;
}
/*
[03/50] incubator-lens git commit: Merge branch 'develop' of
github.com:InMobi/grill into develop
Posted by am...@apache.org.
Merge branch 'develop' of github.com:InMobi/grill into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/699d4c89
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/699d4c89
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/699d4c89
Branch: refs/heads/toapache
Commit: 699d4c8912ff8993831bc9822fa63f8a6302034c
Parents: cea15f8 76c828f
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Thu Nov 6 13:07:56 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Thu Nov 6 13:07:56 2014 +0530
----------------------------------------------------------------------
.../src/main/resources/dim3-storage-tables.xml | 2 +-
.../src/main/resources/dim4-storage-tables.xml | 2 +-
.../main/resources/local-cluster-storage.xml | 2 +-
lens-ml-lib/src/test/resources/lens-site.xml | 4 +-
.../server/api/events/LensEventService.java | 8 ---
.../apache/lens/server/EventServiceImpl.java | 54 +++-------------
.../apache/lens/server/query/LensServerDAO.java | 2 +-
.../server/query/QueryExecutionServiceImpl.java | 2 +-
.../lens/server/query/TestEventService.java | 66 ++++++++++++++++----
.../lens/server/user/TestUserConfigLoader.java | 16 +++--
.../org/apache/lens/storage/db/DBStorage.java | 14 ++---
11 files changed, 88 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
[21/50] incubator-lens git commit: redundant confs removed
Posted by am...@apache.org.
redundant confs removed
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/a23f0659
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/a23f0659
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/a23f0659
Branch: refs/heads/toapache
Commit: a23f0659a36983fdd2b715c992a38d176e4d4ad2
Parents: 4ffc235
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Mon Nov 10 18:01:44 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Mon Nov 10 18:01:44 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/server/api/LensConfConstants.java | 12 ------------
1 file changed, 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a23f0659/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 3b7d7b0..d466be5 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -185,18 +185,6 @@ public class LensConfConstants {
/** The Constant USER_RESOLVER_PROPERTYBASED_FILENAME. */
public static final String USER_RESOLVER_PROPERTYBASED_FILENAME = SERVER_PFX + "user.resolver.propertybased.filename";
- /** The Constant USER_RESOLVER_DB_DRIVER_NAME. */
- public static final String USER_RESOLVER_DB_DRIVER_NAME = SERVER_PFX + "user.resolver.db.driver.name";
-
- /** The Constant USER_RESOLVER_DB_JDBC_URL. */
- public static final String USER_RESOLVER_DB_JDBC_URL = SERVER_PFX + "user.resolver.db.jdbc.url";
-
- /** The Constant USER_RESOLVER_DB_JDBC_USERNAME. */
- public static final String USER_RESOLVER_DB_JDBC_USERNAME = SERVER_PFX + "user.resolver.db.jdbc.username";
-
- /** The Constant USER_RESOLVER_DB_JDBC_PASSWORD. */
- public static final String USER_RESOLVER_DB_JDBC_PASSWORD = SERVER_PFX + "user.resolver.db.jdbc.password";
-
/** The Constant USER_RESOLVER_DB_KEYS. */
public static final String USER_RESOLVER_DB_KEYS = SERVER_PFX + "user.resolver.db.keys";
[20/50] incubator-lens git commit: Merge pull request #170 from
InMobi/svc_stop_npe
Posted by am...@apache.org.
Merge pull request #170 from InMobi/svc_stop_npe
Fix NPE in stop() when init() fails
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/4ffc235f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/4ffc235f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/4ffc235f
Branch: refs/heads/toapache
Commit: 4ffc235f4cbbb290dcf5e07da84185ec1aaa8437
Parents: 480457a e71126c
Author: Jaideep Dhok <ja...@gmail.com>
Authored: Mon Nov 10 12:18:47 2014 +0530
Committer: Jaideep Dhok <ja...@gmail.com>
Committed: Mon Nov 10 12:18:47 2014 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/server/LensServices.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[45/50] incubator-lens git commit: Set next release number to
1.2.0-SNAPSHOT
Posted by am...@apache.org.
Set next release number to 1.2.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/5d15a2bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/5d15a2bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/5d15a2bd
Branch: refs/heads/toapache
Commit: 5d15a2bdd317e2662a05203db6f52dda9db440c4
Parents: b5c5a92
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 13 14:54:41 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 13 14:54:41 2014 +0530
----------------------------------------------------------------------
lens-api/pom.xml | 2 +-
lens-cli/pom.xml | 2 +-
lens-client-dist/pom.xml | 2 +-
lens-client/pom.xml | 2 +-
lens-cube/pom.xml | 2 +-
lens-dist/pom.xml | 2 +-
lens-driver-hive/pom.xml | 2 +-
lens-driver-impala/pom.xml | 2 +-
lens-driver-jdbc/pom.xml | 2 +-
lens-examples/pom.xml | 2 +-
lens-ml-lib/pom.xml | 2 +-
lens-query-lib/pom.xml | 2 +-
lens-server-api/pom.xml | 2 +-
lens-server/pom.xml | 2 +-
lens-storage-db/pom.xml | 2 +-
pom.xml | 2 +-
16 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-api/pom.xml b/lens-api/pom.xml
index dc8f0ca..34e3865 100644
--- a/lens-api/pom.xml
+++ b/lens-api/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-cli/pom.xml
----------------------------------------------------------------------
diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml
index f245d5d..2f74a93 100644
--- a/lens-cli/pom.xml
+++ b/lens-cli/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-client-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-client-dist/pom.xml b/lens-client-dist/pom.xml
index 06348c1..d821229 100644
--- a/lens-client-dist/pom.xml
+++ b/lens-client-dist/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-client-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-client/pom.xml
----------------------------------------------------------------------
diff --git a/lens-client/pom.xml b/lens-client/pom.xml
index 59e94ab..97c56e7 100644
--- a/lens-client/pom.xml
+++ b/lens-client/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-cube/pom.xml
----------------------------------------------------------------------
diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml
index 122c5c8..a1e6be5 100644
--- a/lens-cube/pom.xml
+++ b/lens-cube/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-cube</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml
index b440353..987839d 100644
--- a/lens-dist/pom.xml
+++ b/lens-dist/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-driver-hive/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml
index b17cf96..5d300c2 100644
--- a/lens-driver-hive/pom.xml
+++ b/lens-driver-hive/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-driver-impala/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-impala/pom.xml b/lens-driver-impala/pom.xml
index fb3e251..21f56f5 100644
--- a/lens-driver-impala/pom.xml
+++ b/lens-driver-impala/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-driver-impala</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-driver-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml
index 8c8301d..44f43d6 100644
--- a/lens-driver-jdbc/pom.xml
+++ b/lens-driver-jdbc/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-driver-jdbc</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-examples/pom.xml
----------------------------------------------------------------------
diff --git a/lens-examples/pom.xml b/lens-examples/pom.xml
index c052a7c..daaa04d 100644
--- a/lens-examples/pom.xml
+++ b/lens-examples/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-ml-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml
index 4a1697f..028b3cd 100644
--- a/lens-ml-lib/pom.xml
+++ b/lens-ml-lib/pom.xml
@@ -27,7 +27,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-query-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-query-lib/pom.xml b/lens-query-lib/pom.xml
index c0817fb..86ab4cd 100644
--- a/lens-query-lib/pom.xml
+++ b/lens-query-lib/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-query-lib</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-server-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server-api/pom.xml b/lens-server-api/pom.xml
index fed1bd7..f03cfd6 100644
--- a/lens-server-api/pom.xml
+++ b/lens-server-api/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-server-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-server/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index 338fa48..42192d9 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/lens-storage-db/pom.xml
----------------------------------------------------------------------
diff --git a/lens-storage-db/pom.xml b/lens-storage-db/pom.xml
index b8b1789..bc0ee3f 100644
--- a/lens-storage-db/pom.xml
+++ b/lens-storage-db/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>lens-storage-db</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d15a2bd/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8158239..cca93b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -47,7 +47,7 @@
<groupId>org.apache.lens</groupId>
<artifactId>lens-parent</artifactId>
- <version>1.1.1-SNAPSHOT</version>
+ <version>1.2.0-SNAPSHOT</version>
<name>Lens</name>
<packaging>pom</packaging>
<description>Unified Analytics Platform</description>
[48/50] incubator-lens git commit: Update apache repo links
Posted by am...@apache.org.
Update apache repo links
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/ea0bfc28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/ea0bfc28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/ea0bfc28
Branch: refs/heads/toapache
Commit: ea0bfc28a7ae4d15852f68dd9fe574d9a7def189
Parents: fc1c88a
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 13 16:36:46 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 13 16:36:46 2014 +0530
----------------------------------------------------------------------
pom.xml | 65 ++++++++++++++++++++++++++++++------------------------------
1 file changed, 32 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ea0bfc28/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cca93b2..e8beee3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,7 +51,7 @@
<name>Lens</name>
<packaging>pom</packaging>
<description>Unified Analytics Platform</description>
- <url>https://github.com/InMobi/grill</url>
+ <url>http://lens.incubator.apache.org</url>
<inceptionYear>2014</inceptionYear>
<licenses>
<license>
@@ -65,27 +65,44 @@
<url>http://www.apache.org</url>
</organization>
+ <issueManagement>
+ <system>JIRA</system>
+ <url>https://issues.apache.org/jira/browse/LENS</url>
+ </issueManagement>
+
+ <ciManagement>
+ <system>Jenkins</system>
+ <url>https://builds.apache.org/view/H-L/view/Lens/</url>
+ </ciManagement>
+
<mailingLists>
<mailingList>
- <name>grill-users@googlegroups.com</name>
- <subscribe>grill-users+subscribe@googlegroups.com</subscribe>
- <unsubscribe>grill-users+unsubscribe@googlegroups.com</unsubscribe>
- <post>grill-users@googlegroups.com</post>
- <archive>https://groups.google.com/forum/?hl=en#!forum/grill-users</archive>
+ <name>lens-user</name>
+ <subscribe>user-subscribe@lens.incubator.apache.org</subscribe>
+ <unsubscribe>user-unsubscribe@lens.incubator.apache.org</unsubscribe>
+ <post>user@lens.incubator.apache.org</post>
+ <archive>http://mail-archives.apache.org/mod_mbox/lens-user/</archive>
</mailingList>
<mailingList>
- <name>grill-dev@googlegroups.com</name>
- <subscribe>grill-dev+subscribe@googlegroups.com</subscribe>
- <unsubscribe>grill-dev+unsubscribe@googlegroups.com</unsubscribe>
- <post>grill-dev@googlegroups.com</post>
- <archive>https://groups.google.com/forum/?hl=en#!forum/grill-dev</archive>
+ <name>lens-dev</name>
+ <subscribe>dev-subscribe@lens.incubator.apache.org</subscribe>
+ <unsubscribe>dev-unsubscribe@lens.incubator.apache.org</unsubscribe>
+ <post>dev@lens.incubator.apache.org</post>
+ <archive>http://mail-archives.apache.org/mod_mbox/lens-dev/</archive>
+ </mailingList>
+ <mailingList>
+ <name>lens-commits</name>
+ <subscribe>commits-subscribe@lens.incubator.apache.org</subscribe>
+ <unsubscribe>commits-unsubscribe@lens.incubator.apache.org</unsubscribe>
+ <post>commits@lens.incubator.apache.org</post>
+ <archive>http://mail-archives.apache.org/mod_mbox/lens-commits/</archive>
</mailingList>
</mailingLists>
<scm>
- <connection>scm:git:https://github.com/InMobi/grill.git</connection>
- <url>https://github.com/InMobi/grill</url>
- <developerConnection>scm:git:https://github.com/InMobi/grill.git</developerConnection>
+ <connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-lens.git</connection>
+ <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-lens.git</developerConnection>
+ <url>https://git-wip-us.apache.org/repos/asf/incubator-falcon.git</url>
<tag>HEAD</tag>
</scm>
@@ -844,26 +861,8 @@
</dependencyManagement>
- <distributionManagement>
- <repository>
- <id>InMobiReleases</id>
- <name>InMobi release Repository</name>
- <url>https://github.com/InMobi/mvn-repo/raw/master/releases</url>
- </repository>
- <snapshotRepository>
- <id>InMobiSnapshots</id>
- <name>InMobi snapshot Repository</name>
- <url>https://github.com/InMobi/mvn-repo/raw/master/snapshots</url>
- </snapshotRepository>
- <site>
- <id>lens-site</id>
- <url>https://inmobi.github.io/grill/</url>
- </site>
- </distributionManagement>
-
-
<modules>
- <module>checkstyle</module>
+ <module>checkstyle</module>
<module>lens-api</module>
<module>lens-server-api</module>
<module>lens-cube</module>
[26/50] incubator-lens git commit: test cases modified to include max
and min
Posted by am...@apache.org.
test cases modified to include max and min
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/77c8ca0c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/77c8ca0c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/77c8ca0c
Branch: refs/heads/toapache
Commit: 77c8ca0cd5cd7c6eb055b6e99bf10ffbac612fad
Parents: 8adf53b
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Tue Nov 11 15:15:02 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Tue Nov 11 15:15:02 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/cube/metadata/TestCubeMetastoreClient.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/77c8ca0c/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index da54937..09dd5b9 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -131,7 +131,7 @@ public class TestCubeMetastoreClient {
private static void defineCube(String cubeName, String cubeNameWithProps, String derivedCubeName,
String derivedCubeNameWithProps) throws ParseException {
cubeMeasures = new HashSet<CubeMeasure>();
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr1", "int", "first measure")));
+ cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr1", "int", "first measure"), null, null, null, null, null, null, null, 0.0, 9999.0));
cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr2", "float", "second measure"), "Measure2", null, "SUM",
"RS"));
cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr3", "double", "third measure"), "Measure3", null, "MAX",
@@ -139,13 +139,13 @@ public class TestCubeMetastoreClient {
cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr4", "bigint", "fourth measure"), "Measure4", null, "COUNT",
null));
cubeMeasures.add(new ColumnMeasure(new FieldSchema("msrstarttime", "int", "measure with start time"),
- "Measure With Starttime", null, null, null, now, null, null));
+ "Measure With Starttime", null, null, null, now, null, null, 0.0, 999999.0));
cubeMeasures.add(new ColumnMeasure(new FieldSchema("msrendtime", "float", "measure with end time"),
"Measure With Endtime", null, "SUM", "RS", now, now, null));
cubeMeasures.add(new ColumnMeasure(new FieldSchema("msrcost", "double", "measure with cost"), "Measure With cost",
null, "MAX", null, now, now, 100.0));
cubeMeasures.add(new ColumnMeasure(new FieldSchema("msrcost2", "bigint", "measure with cost"),
- "Measure With cost2", null, "MAX", null, null, null, 100.0));
+ "Measure With cost2", null, "MAX", null, null, null, 100.0, 0.0, 999999999999999999999999999.0));
cubeDimensions = new HashSet<CubeDimAttribute>();
List<CubeDimAttribute> locationHierarchy = new ArrayList<CubeDimAttribute>();
[13/50] incubator-lens git commit: Ignore exception if example table
already created
Posted by am...@apache.org.
Ignore exception if example table already created
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/31d41d91
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/31d41d91
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/31d41d91
Branch: refs/heads/toapache
Commit: 31d41d91902cf0124c7d9dbbac3dc9316f464193
Parents: a97d0d0
Author: jdhok <ja...@inmobi.com>
Authored: Fri Nov 7 14:02:47 2014 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Fri Nov 7 14:02:47 2014 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/ml/ExampleUtils.java | 2 +-
.../src/test/java/org/apache/lens/ml/TestMLResource.java | 9 +++++++--
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/31d41d91/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
index a48dcc2..8ca2b7a 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/ExampleUtils.java
@@ -92,7 +92,7 @@ public class ExampleUtils {
partCols.add(new FieldSchema("dummy_partition_col", "string", ""));
tbl.setPartCols(partCols);
- Hive.get(conf).createTable(tbl);
+ Hive.get(conf).createTable(tbl, false);
LOG.info("Created table " + tableName);
// Add partition for the data file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/31d41d91/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
index 4abc010..8ebff9c 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
@@ -30,6 +30,7 @@ import javax.ws.rs.core.Application;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hive.service.Service;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.client.LensConnectionParams;
@@ -180,8 +181,12 @@ public class TestMLResource extends LensJerseyTest {
LOG.info("Creating training table from file " + sampleDataFileURI.toString());
Map<String, String> tableParams = new HashMap<String, String>();
- ExampleUtils.createTable(conf, database, tableName, sampleDataFileURI.toString(), labelColumn, tableParams,
- features);
+ try {
+ ExampleUtils.createTable(conf, database, tableName, sampleDataFileURI.toString(), labelColumn, tableParams,
+ features);
+ } catch (HiveException exc) {
+ exc.printStackTrace();
+ }
MLTask.Builder taskBuilder = new MLTask.Builder();
taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn).outputTable(outputTable)
[12/50] incubator-lens git commit: Removing CubeDriver
Posted by am...@apache.org.
Removing CubeDriver
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/dbe81435
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/dbe81435
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/dbe81435
Branch: refs/heads/toapache
Commit: dbe8143500f2d2cd58f5fed7de3b06b3af9a3002
Parents: 5bce5c4
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Fri Nov 7 13:19:38 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Fri Nov 7 13:19:38 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/cube/CubeDriver.java | 604 -------------------
.../apache/lens/driver/cube/RewriteUtil.java | 26 +-
.../apache/lens/driver/cube/TestCubeDriver.java | 174 ------
.../lens/driver/cube/TestMinCostSelector.java | 2 +-
.../server/api/driver/MinQueryCostSelector.java | 83 +++
.../server/query/QueryExecutionServiceImpl.java | 3 +-
6 files changed, 99 insertions(+), 793 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dbe81435/lens-cube/src/main/java/org/apache/lens/driver/cube/CubeDriver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/CubeDriver.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/CubeDriver.java
deleted file mode 100644
index 3d548d6..0000000
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/CubeDriver.java
+++ /dev/null
@@ -1,604 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.driver.cube;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.lens.api.LensException;
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryPrepareHandle;
-import org.apache.lens.api.query.QueryStatus;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.driver.*;
-import org.apache.lens.server.api.events.LensEventListener;
-import org.apache.lens.server.api.query.PreparedQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
-import org.apache.log4j.Logger;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-/**
- * The Class CubeDriver.
- */
-public class CubeDriver implements LensDriver {
-
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(CubeDriver.class);
-
- /** The drivers. */
- private final List<LensDriver> drivers;
-
- /** The driver selector. */
- private final DriverSelector driverSelector;
-
- /** The conf. */
- private Configuration conf;
-
- /** The query contexts. */
- private Map<QueryHandle, QueryContext> queryContexts = new HashMap<QueryHandle, QueryContext>();
-
- /** The prepared queries. */
- private Map<QueryPrepareHandle, PreparedQueryContext> preparedQueries = new HashMap<QueryPrepareHandle, PreparedQueryContext>();
-
- /**
- * Instantiates a new cube driver.
- *
- * @param conf
- * the conf
- * @throws LensException
- * the lens exception
- */
- public CubeDriver(Configuration conf) throws LensException {
- this(conf, new MinQueryCostSelector());
- }
-
- /**
- * Instantiates a new cube driver.
- *
- * @param conf
- * the conf
- * @param driverSelector
- * the driver selector
- * @throws LensException
- * the lens exception
- */
- public CubeDriver(Configuration conf, DriverSelector driverSelector) throws LensException {
- this.conf = new HiveConf(conf, CubeDriver.class);
- this.drivers = new ArrayList<LensDriver>();
- loadDrivers();
- this.driverSelector = driverSelector;
- }
-
- /**
- * Load drivers.
- *
- * @throws LensException
- * the lens exception
- */
- private void loadDrivers() throws LensException {
- String[] driverClasses = conf.getStrings(LensConfConstants.DRIVER_CLASSES);
- if (driverClasses != null) {
- for (String driverClass : driverClasses) {
- try {
- Class<?> clazz = Class.forName(driverClass);
- LensDriver driver = (LensDriver) clazz.newInstance();
- driver.configure(conf);
- drivers.add(driver);
- LOG.info("Cube driver loaded driver " + driverClass);
- } catch (Exception e) {
- LOG.warn("Could not load the driver:" + driverClass, e);
- throw new LensException("Could not load driver " + driverClass, e);
- }
- }
- } else {
- throw new LensException("No drivers specified");
- }
- }
-
- /**
- * Select driver.
- *
- * @param queries
- * the queries
- * @param conf
- * the conf
- * @return the lens driver
- */
- protected LensDriver selectDriver(Map<LensDriver, String> queries, Configuration conf) {
- return driverSelector.select(drivers, queries, conf);
- }
-
- /**
- * The Class MinQueryCostSelector.
- */
- public static class MinQueryCostSelector implements DriverSelector {
-
- /**
- * Returns the driver that has the minimum query cost.
- *
- * @param drivers
- * the drivers
- * @param driverQueries
- * the driver queries
- * @param conf
- * the conf
- * @return the lens driver
- */
- @Override
- public LensDriver select(Collection<LensDriver> drivers, final Map<LensDriver, String> driverQueries,
- final Configuration conf) {
- return Collections.min(drivers, new Comparator<LensDriver>() {
- @Override
- public int compare(LensDriver d1, LensDriver d2) {
- DriverQueryPlan c1 = null;
- DriverQueryPlan c2 = null;
- // Handle cases where the queries can be null because the storages are not
- // supported.
- if (driverQueries.get(d1) == null) {
- return 1;
- }
- if (driverQueries.get(d2) == null) {
- return -1;
- }
- try {
- c1 = d1.explain(driverQueries.get(d1), conf);
- } catch (LensException e) {
- LOG.warn("Explain query:" + driverQueries.get(d1) + " on Driver:" + d1.getClass().getSimpleName()
- + " failed", e);
- }
- try {
- c2 = d2.explain(driverQueries.get(d2), conf);
- } catch (LensException e) {
- LOG.warn("Explain query:" + driverQueries.get(d2) + " on Driver:" + d2.getClass().getSimpleName()
- + " failed", e);
- }
- if (c1 == null && c2 == null) {
- return 0;
- } else if (c1 == null && c2 != null) {
- return 1;
- } else if (c1 != null && c2 == null) {
- return -1;
- }
- return c1.getCost().compareTo(c2.getCost());
- }
- });
- }
- }
-
- /**
- * Execute.
- *
- * @param query
- * the query
- * @param conf
- * the conf
- * @return the lens result set
- * @throws LensException
- * the lens exception
- */
- public LensResultSet execute(String query, Configuration conf) throws LensException {
- QueryContext ctx = createQueryContext(query, conf);
- LensResultSet result = execute(ctx);
- queryContexts.remove(ctx.getQueryHandle());
- return result;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#execute(org.apache.lens.server.api.query.QueryContext)
- */
- @Override
- public LensResultSet execute(QueryContext ctx) throws LensException {
- rewriteAndSelect(ctx);
- return ctx.getSelectedDriver().execute(ctx);
- }
-
- /**
- * Rewrite and select.
- *
- * @param ctx
- * the ctx
- * @throws LensException
- * the lens exception
- */
- private void rewriteAndSelect(QueryContext ctx) throws LensException {
- queryContexts.put(ctx.getQueryHandle(), ctx);
- Map<LensDriver, String> driverQueries = RewriteUtil.rewriteQuery(ctx.getUserQuery(), drivers, ctx.getConf());
-
- // 2. select driver to run the query
- LensDriver driver = selectDriver(driverQueries, conf);
-
- ctx.setSelectedDriver(driver);
- ctx.setDriverQuery(driverQueries.get(driver));
- }
-
- /**
- * Creates the query context.
- *
- * @param query
- * the query
- * @param conf
- * the conf
- * @return the query context
- */
- private QueryContext createQueryContext(String query, Configuration conf) {
- return new QueryContext(query, null, conf);
- }
-
- /**
- * Execute async.
- *
- * @param query
- * the query
- * @param conf
- * the conf
- * @return the query handle
- * @throws LensException
- * the lens exception
- */
- public QueryHandle executeAsync(String query, Configuration conf) throws LensException {
- QueryContext ctx = createQueryContext(query, conf);
- executeAsync(ctx);
- return ctx.getQueryHandle();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#executeAsync(org.apache.lens.server.api.query.QueryContext)
- */
- @Override
- public void executeAsync(QueryContext ctx) throws LensException {
- rewriteAndSelect(ctx);
- ctx.getSelectedDriver().executeAsync(ctx);
- }
-
- /**
- * Gets the status.
- *
- * @param handle
- * the handle
- * @return the status
- * @throws LensException
- * the lens exception
- */
- public QueryStatus getStatus(QueryHandle handle) throws LensException {
- updateStatus(getContext(handle));
- QueryStatus status = getContext(handle).getDriverStatus().toQueryStatus();
- if (status.getStatus().equals(QueryStatus.Status.EXECUTED)) {
- return DriverQueryStatus.createQueryStatus(QueryStatus.Status.SUCCESSFUL, getContext(handle).getDriverStatus());
- }
- return status;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#updateStatus(org.apache.lens.server.api.query.QueryContext)
- */
- public void updateStatus(QueryContext context) throws LensException {
- context.getSelectedDriver().updateStatus(context);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#fetchResultSet(org.apache.lens.server.api.query.QueryContext)
- */
- @Override
- public LensResultSet fetchResultSet(QueryContext context) throws LensException {
- return context.getSelectedDriver().fetchResultSet(context);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#configure(org.apache.hadoop.conf.Configuration)
- */
- @Override
- public void configure(Configuration conf) {
- this.conf = conf;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#cancelQuery(org.apache.lens.api.query.QueryHandle)
- */
- @Override
- public boolean cancelQuery(QueryHandle handle) throws LensException {
- return getContext(handle).getSelectedDriver().cancelQuery(handle);
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#close()
- */
- @Override
- public void close() throws LensException {
- for (LensDriver driver : drivers) {
- driver.close();
- }
- drivers.clear();
- queryContexts.clear();
- }
-
- /**
- * Add a listener for driver events.
- *
- * @param driverEventListener
- * the driver event listener
- */
- @Override
- public void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener) {
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#closeQuery(org.apache.lens.api.query.QueryHandle)
- */
- @Override
- public void closeQuery(QueryHandle handle) throws LensException {
- getContext(handle).getSelectedDriver().closeQuery(handle);
- queryContexts.remove(handle);
- }
-
- /**
- * Gets the context.
- *
- * @param handle
- * the handle
- * @return the context
- * @throws LensException
- * the lens exception
- */
- private QueryContext getContext(QueryHandle handle) throws LensException {
- QueryContext ctx = queryContexts.get(handle);
- if (ctx == null) {
- throw new LensException("Query not found " + ctx);
- }
- return ctx;
- }
-
- public List<LensDriver> getDrivers() {
- return drivers;
- }
-
- /**
- * Rewrite and select for prepare.
- *
- * @param ctx
- * the ctx
- * @throws LensException
- * the lens exception
- */
- private void rewriteAndSelectForPrepare(PreparedQueryContext ctx) throws LensException {
- preparedQueries.put(ctx.getPrepareHandle(), ctx);
- Map<LensDriver, String> driverQueries = RewriteUtil.rewriteQuery(ctx.getUserQuery(), drivers, ctx.getConf());
-
- // 2. select driver to run the query
- LensDriver driver = selectDriver(driverQueries, conf);
-
- ctx.setSelectedDriver(driver);
- ctx.setDriverQuery(driverQueries.get(driver));
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#explain(java.lang.String, org.apache.hadoop.conf.Configuration)
- */
- @Override
- public DriverQueryPlan explain(String query, Configuration conf) throws LensException {
- Map<LensDriver, String> driverQueries = RewriteUtil.rewriteQuery(query, drivers, conf);
- LensDriver driver = selectDriver(driverQueries, conf);
- return driver.explain(driverQueries.get(driver), conf);
- }
-
- /**
- * Execute prepare.
- *
- * @param handle
- * the handle
- * @param conf
- * the conf
- * @return the lens result set
- * @throws LensException
- * the lens exception
- */
- @Deprecated
- public LensResultSet executePrepare(QueryHandle handle, Configuration conf) throws LensException {
- QueryPrepareHandle pHandle = new QueryPrepareHandle(handle.getHandleId());
- QueryContext ctx = new QueryContext(preparedQueries.get(pHandle), null, conf);
- ctx.setQueryHandle(handle);
- return execute(ctx);
- }
-
- /**
- * Execute prepare async.
- *
- * @param handle
- * the handle
- * @param conf
- * the conf
- * @throws LensException
- * the lens exception
- */
- @Deprecated
- public void executePrepareAsync(QueryHandle handle, Configuration conf) throws LensException {
- QueryPrepareHandle pHandle = new QueryPrepareHandle(handle.getHandleId());
- QueryContext ctx = new QueryContext(preparedQueries.get(pHandle), null, conf);
- ctx.setQueryHandle(handle);
- executeAsync(ctx);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#prepare(org.apache.lens.server.api.query.PreparedQueryContext)
- */
- @Override
- public void prepare(PreparedQueryContext pContext) throws LensException {
- rewriteAndSelectForPrepare(pContext);
- pContext.getSelectedDriver().prepare(pContext);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.lens.server.api.driver.LensDriver#explainAndPrepare(org.apache.lens.server.api.query.PreparedQueryContext
- * )
- */
- @Override
- public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws LensException {
- LOG.info("In explainAndPrepare, preparing :" + pContext.getUserQuery());
- rewriteAndSelectForPrepare(pContext);
- return pContext.getSelectedDriver().explainAndPrepare(pContext);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#closePreparedQuery(org.apache.lens.api.query.QueryPrepareHandle)
- */
- @Override
- public void closePreparedQuery(QueryPrepareHandle handle) throws LensException {
- PreparedQueryContext ctx = preparedQueries.remove(handle);
- if (ctx != null) {
- ctx.getSelectedDriver().closePreparedQuery(handle);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.LensDriver#closeResultSet(org.apache.lens.api.query.QueryHandle)
- */
- @Override
- public void closeResultSet(QueryHandle handle) throws LensException {
- getContext(handle).getSelectedDriver().closeResultSet(handle);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.lens.server.api.driver.LensDriver#registerForCompletionNotification(org.apache.lens.api.query.QueryHandle
- * , long, org.apache.lens.server.api.driver.QueryCompletionListener)
- */
- @Override
- public void registerForCompletionNotification(QueryHandle handle, long timeoutMillis, QueryCompletionListener listener)
- throws LensException {
- throw new LensException("Not implemented");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.io.Externalizable#readExternal(java.io.ObjectInput)
- */
- @Override
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- drivers.clear();
- Map<String, LensDriver> driverMap = new HashMap<String, LensDriver>();
- synchronized (drivers) {
- drivers.clear();
- int numDrivers = in.readInt();
- for (int i = 0; i < numDrivers; i++) {
- String driverClsName = in.readUTF();
- LensDriver driver;
- try {
- Class<? extends LensDriver> driverCls = (Class<? extends LensDriver>) Class.forName(driverClsName);
- driver = (LensDriver) driverCls.newInstance();
- driver.configure(conf);
- } catch (Exception e) {
- LOG.error("Could not instantiate driver:" + driverClsName);
- throw new IOException(e);
- }
- driver.readExternal(in);
- drivers.add(driver);
- driverMap.put(driverClsName, driver);
- }
- }
-
- synchronized (queryContexts) {
- int numQueries = in.readInt();
- for (int i = 0; i < numQueries; i++) {
- QueryContext ctx = (QueryContext) in.readObject();
- queryContexts.put(ctx.getQueryHandle(), ctx);
- boolean driverAvailable = in.readBoolean();
- if (driverAvailable) {
- String clsName = in.readUTF();
- ctx.setSelectedDriver(driverMap.get(clsName));
- }
- }
- LOG.info("Recovered " + queryContexts.size() + " queries");
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput)
- */
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- // persist all drivers
- synchronized (drivers) {
- out.writeInt(drivers.size());
- for (LensDriver driver : drivers) {
- out.writeUTF(driver.getClass().getName());
- driver.writeExternal(out);
- }
- }
- // persist allQueries
- synchronized (queryContexts) {
- out.writeInt(queryContexts.size());
- for (QueryContext ctx : queryContexts.values()) {
- out.writeObject(ctx);
- boolean isDriverAvailable = (ctx.getSelectedDriver() != null);
- out.writeBoolean(isDriverAvailable);
- if (isDriverAvailable) {
- out.writeUTF(ctx.getSelectedDriver().getClass().getName());
- }
- }
- }
- LOG.info("Persisted " + queryContexts.size() + " queries");
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dbe81435/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
index 79de368..7f83823 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
@@ -36,11 +36,13 @@ import org.apache.lens.cube.parse.CubeQueryConfUtil;
import org.apache.lens.cube.parse.CubeQueryRewriter;
import org.apache.lens.cube.parse.HQLParser;
import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.log4j.Logger;
/**
* The Class RewriteUtil.
*/
public class RewriteUtil {
+ public static final Logger LOG = Logger.getLogger(RewriteUtil.class);
/** The cube pattern. */
static Pattern cubePattern = Pattern.compile(".*CUBE(.*)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE
@@ -80,7 +82,7 @@ public class RewriteUtil {
*/
static List<CubeQueryInfo> findCubePositions(String query) throws SemanticException, ParseException {
ASTNode ast = HQLParser.parseHQL(query);
- CubeDriver.LOG.debug("User query AST:" + ast.dump());
+ LOG.debug("User query AST:" + ast.dump());
List<CubeQueryInfo> cubeQueries = new ArrayList<CubeQueryInfo>();
findCubePositions(ast, cubeQueries, query);
for (CubeQueryInfo cqi : cubeQueries) {
@@ -106,12 +108,12 @@ public class RewriteUtil {
int child_count = ast.getChildCount();
if (ast.getToken() != null) {
if (ast.getChild(0) != null) {
- CubeDriver.LOG.debug("First child:" + ast.getChild(0) + " Type:"
+ LOG.debug("First child:" + ast.getChild(0) + " Type:"
+ ((ASTNode) ast.getChild(0)).getToken().getType());
}
if (ast.getToken().getType() == HiveParser.TOK_QUERY
&& ((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.KW_CUBE) {
- CubeDriver.LOG.debug("Inside cube clause");
+ LOG.debug("Inside cube clause");
CubeQueryInfo cqi = new CubeQueryInfo();
cqi.cubeAST = ast;
if (ast.getParent() != null) {
@@ -132,7 +134,7 @@ public class RewriteUtil {
cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine() - 1, "UNION ALL");
} else {
// Not expected to reach here
- CubeDriver.LOG.warn("Unknown query pattern found with AST:" + ast.dump());
+ LOG.warn("Unknown query pattern found with AST:" + ast.dump());
throw new SemanticException("Unknown query pattern");
}
} else {
@@ -140,12 +142,12 @@ public class RewriteUtil {
// one for next AST
// and one for the close parenthesis if there are no more unionall
// or one for the string 'UNION ALL' if there are more union all
- CubeDriver.LOG.debug("Child of union all");
+ LOG.debug("Child of union all");
cqi.endPos = getEndPos(originalQuery, parent.getParent().getChild(1).getCharPositionInLine(), ")",
"UNION ALL");
}
}
- CubeDriver.LOG.debug("Adding cqi " + cqi + " query:" + originalQuery.substring(cqi.startPos, cqi.endPos));
+ LOG.debug("Adding cqi " + cqi + " query:" + originalQuery.substring(cqi.startPos, cqi.endPos));
cubeQueries.add(cqi);
} else {
for (int child_pos = 0; child_pos < child_count; ++child_pos) {
@@ -153,7 +155,7 @@ public class RewriteUtil {
}
}
} else {
- CubeDriver.LOG.warn("Null AST!");
+ LOG.warn("Null AST!");
}
}
@@ -197,7 +199,7 @@ public class RewriteUtil {
Configuration conf = new Configuration(driver.getConf());
for (Map.Entry<String, String> entry : queryConf) {
if (entry.getKey().equals(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES)) {
- CubeDriver.LOG.warn(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES + " value : " + entry.getValue()
+ LOG.warn(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES + " value : " + entry.getValue()
+ " from query conf ignored/");
continue;
}
@@ -268,21 +270,21 @@ public class RewriteUtil {
int start = 0;
try {
for (RewriteUtil.CubeQueryInfo cqi : cubeQueries) {
- CubeDriver.LOG.debug("Rewriting cube query:" + cqi.query);
+ LOG.debug("Rewriting cube query:" + cqi.query);
if (start != cqi.startPos) {
builder.append(replacedQuery.substring(start, cqi.startPos));
}
String hqlQuery = rewriter.rewrite(cqi.query).toHQL();
- CubeDriver.LOG.debug("Rewritten query:" + hqlQuery);
+ LOG.debug("Rewritten query:" + hqlQuery);
builder.append(hqlQuery);
start = cqi.endPos;
}
builder.append(replacedQuery.substring(start));
String finalQuery = builder.toString();
- CubeDriver.LOG.info("Final rewritten query for driver:" + driver + " is: " + finalQuery);
+ LOG.info("Final rewritten query for driver:" + driver + " is: " + finalQuery);
driverQueries.put(driver, finalQuery);
} catch (SemanticException e) {
- CubeDriver.LOG.warn("Driver : " + driver.getClass().getName() + " Skipped for the query rewriting due to "
+ LOG.warn("Driver : " + driver.getClass().getName() + " Skipped for the query rewriting due to "
+ e.getMessage());
rewriteFailure.append(" Driver :").append(driver.getClass().getName());
rewriteFailure.append(" Cause :" + e.getLocalizedMessage());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dbe81435/lens-cube/src/test/java/org/apache/lens/driver/cube/TestCubeDriver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestCubeDriver.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestCubeDriver.java
deleted file mode 100644
index 6a84a59..0000000
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestCubeDriver.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.driver.cube;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.util.Arrays;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.lens.api.LensException;
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-import org.apache.lens.driver.cube.CubeDriver;
-import org.apache.lens.server.api.driver.DriverQueryPlan;
-import org.apache.lens.server.api.driver.LensDriver;
-import org.apache.lens.server.api.driver.LensResultSet;
-import org.testng.Assert;
-import org.testng.annotations.AfterTest;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Test;
-
-/**
- * The Class TestCubeDriver.
- */
-public class TestCubeDriver {
-
- /** The conf. */
- Configuration conf = new Configuration();
-
- /** The cube driver. */
- CubeDriver cubeDriver;
-
- /**
- * Before test.
- *
- * @throws Exception
- * the exception
- */
- @BeforeTest
- public void beforeTest() throws Exception {
- cubeDriver = new CubeDriver(conf);
- conf.setInt("mock.driver.test.val", 5);
- }
-
- /**
- * After test.
- *
- * @throws Exception
- * the exception
- */
- @AfterTest
- public void afterTest() throws Exception {
- }
-
- /**
- * Test cube driver.
- *
- * @throws Exception
- * the exception
- */
- @Test
- public void testCubeDriver() throws Exception {
- String addQ = "add jar xyz.jar";
- LensResultSet result = cubeDriver.execute(addQ, conf);
- Assert.assertNotNull(result);
- Assert.assertEquals(((MockDriver) cubeDriver.getDrivers().get(0)).query, addQ);
-
- String setQ = "set xyz=random";
- result = cubeDriver.execute(setQ, conf);
- Assert.assertNotNull(result);
- Assert.assertEquals(((MockDriver) cubeDriver.getDrivers().get(0)).query, setQ);
-
- String query = "select name from table";
- DriverQueryPlan plan = cubeDriver.explain(query, conf);
- String planString = plan.getPlan();
- Assert.assertEquals(query, planString);
-
- result = cubeDriver.execute(query, conf);
- Assert.assertNotNull(result);
- Assert.assertNotNull(result.getMetadata());
-
- QueryHandle handle = cubeDriver.executeAsync(query, conf);
- Assert.assertEquals(cubeDriver.getStatus(handle).getStatus(), QueryStatus.Status.SUCCESSFUL);
- Assert.assertFalse(cubeDriver.cancelQuery(handle));
-
- cubeDriver.closeQuery(handle);
-
- Throwable th = null;
- try {
- cubeDriver.getStatus(handle);
- } catch (LensException e) {
- th = e;
- }
- Assert.assertNotNull(th);
- }
-
- /**
- * Test cube driver read write.
- *
- * @throws Exception
- * the exception
- */
- @Test
- public void testCubeDriverReadWrite() throws Exception {
- // Test read/write for cube driver
- CubeDriver cubeDriver = new CubeDriver(conf);
- ByteArrayOutputStream driverOut = new ByteArrayOutputStream();
- ObjectOutputStream out = new ObjectOutputStream(driverOut);
- cubeDriver.writeExternal(out);
- out.close();
- System.out.println(Arrays.toString(driverOut.toByteArray()));
-
- ByteArrayInputStream driverIn = new ByteArrayInputStream(driverOut.toByteArray());
- conf.setInt("mock.driver.test.val", -1);
- CubeDriver newDriver = new CubeDriver(conf);
- newDriver.readExternal(new ObjectInputStream(driverIn));
- driverIn.close();
- Assert.assertEquals(newDriver.getDrivers().size(), cubeDriver.getDrivers().size());
-
- for (LensDriver driver : newDriver.getDrivers()) {
- if (driver instanceof MockDriver) {
- MockDriver md = (MockDriver) driver;
- Assert.assertEquals(md.getTestIOVal(), 5);
- }
- }
- }
-
- /**
- * Test cube driver restart.
- *
- * @throws Exception
- * the exception
- */
- @Test
- public void testCubeDriverRestart() throws Exception {
- // Test read/write for cube driver
- CubeDriver cubeDriver = new CubeDriver(conf);
- String query = "select name from table";
- QueryHandle handle = cubeDriver.executeAsync(query, conf);
-
- ByteArrayOutputStream driverOut = new ByteArrayOutputStream();
- ObjectOutputStream out = new ObjectOutputStream(driverOut);
- cubeDriver.writeExternal(out);
- out.close();
- System.out.println(Arrays.toString(driverOut.toByteArray()));
-
- ByteArrayInputStream driverIn = new ByteArrayInputStream(driverOut.toByteArray());
- CubeDriver newDriver = new CubeDriver(conf);
- newDriver.readExternal(new ObjectInputStream(driverIn));
- driverIn.close();
- Assert.assertEquals(newDriver.getDrivers().size(), cubeDriver.getDrivers().size());
-
- Assert.assertEquals(cubeDriver.getStatus(handle).getStatus(), QueryStatus.Status.SUCCESSFUL);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dbe81435/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
index 5d9bd4b..66de80c 100644
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
+++ b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
@@ -25,9 +25,9 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.lens.api.LensException;
-import org.apache.lens.driver.cube.CubeDriver.MinQueryCostSelector;
import org.apache.lens.server.api.driver.DriverQueryPlan;
import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.driver.MinQueryCostSelector;
import org.testng.Assert;
import org.testng.annotations.Test;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dbe81435/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
new file mode 100644
index 0000000..a280c1b
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.driver;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.lens.api.LensException;
+import org.apache.log4j.Logger;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Map;
+
+public class MinQueryCostSelector implements DriverSelector {
+ public static final Logger LOG = Logger.getLogger(MinQueryCostSelector.class);
+
+ /**
+ * Returns the driver that has the minimum query cost.
+ *
+ * @param drivers
+ * the drivers
+ * @param driverQueries
+ * the driver queries
+ * @param conf
+ * the conf
+ * @return the lens driver
+ */
+ @Override
+ public LensDriver select(Collection<LensDriver> drivers, final Map<LensDriver, String> driverQueries,
+ final Configuration conf) {
+ return Collections.min(drivers, new Comparator<LensDriver>() {
+ @Override
+ public int compare(LensDriver d1, LensDriver d2) {
+ DriverQueryPlan c1 = null;
+ DriverQueryPlan c2 = null;
+ // Handle cases where the queries can be null because the storages are not
+ // supported.
+ if (driverQueries.get(d1) == null) {
+ return 1;
+ }
+ if (driverQueries.get(d2) == null) {
+ return -1;
+ }
+ try {
+ c1 = d1.explain(driverQueries.get(d1), conf);
+ } catch (LensException e) {
+ LOG.warn("Explain query:" + driverQueries.get(d1) + " on Driver:" + d1.getClass().getSimpleName()
+ + " failed", e);
+ }
+ try {
+ c2 = d2.explain(driverQueries.get(d2), conf);
+ } catch (LensException e) {
+ LOG.warn("Explain query:" + driverQueries.get(d2) + " on Driver:" + d2.getClass().getSimpleName()
+ + " failed", e);
+ }
+ if (c1 == null && c2 == null) {
+ return 0;
+ } else if (c1 == null && c2 != null) {
+ return 1;
+ } else if (c1 != null && c2 == null) {
+ return -1;
+ }
+ return c1.getCost().compareTo(c2.getCost());
+ }
+ });
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dbe81435/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 5952085..230b403 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -66,7 +66,6 @@ import org.apache.lens.api.query.QueryResultSetMetadata;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.api.query.SubmitOp;
import org.apache.lens.api.query.QueryStatus.Status;
-import org.apache.lens.driver.cube.CubeDriver;
import org.apache.lens.driver.cube.RewriteUtil;
import org.apache.lens.driver.hive.HiveDriver;
import org.apache.lens.server.LensService;
@@ -249,7 +248,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
} else {
throw new LensException("No drivers specified");
}
- driverSelector = new CubeDriver.MinQueryCostSelector();
+ driverSelector = new MinQueryCostSelector();
}
protected LensEventService getEventService() {
[02/50] incubator-lens git commit: Merge pull request #163 from
InMobi/fix_insert_finished_query_test
Posted by am...@apache.org.
Merge pull request #163 from InMobi/fix_insert_finished_query_test
Don't drop all tables in TestUserConfigLoader
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/76c828ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/76c828ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/76c828ff
Branch: refs/heads/toapache
Commit: 76c828ff82a19bd2b9d424831c57ee705e0ca5cd
Parents: cca9b74 65d9036
Author: Amareshwari <am...@inmobi.com>
Authored: Wed Nov 5 15:33:25 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Wed Nov 5 15:33:25 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/server/query/LensServerDAO.java | 2 +-
.../lens/server/user/TestUserConfigLoader.java | 16 ++++++++++------
2 files changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[18/50] incubator-lens git commit: fix test failure in
TestMetastoreService
Posted by am...@apache.org.
fix test failure in TestMetastoreService
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/480457a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/480457a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/480457a4
Branch: refs/heads/toapache
Commit: 480457a427a8ee8606ecd101527380ba24ad0124
Parents: 27cab4a
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Fri Nov 7 16:26:42 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Fri Nov 7 16:27:17 2014 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/server/metastore/TestMetastoreService.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/480457a4/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 7ca955b..af8f93c 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -1678,6 +1678,7 @@ public class TestMetastoreService extends LensJerseyTest {
setCurrentDatabase(DB);
createStorage("S1");
createStorage("S2");
+ createStorage("test");
try {
createDimTable(table);
[32/50] incubator-lens git commit: Merge pull request #165 from
InMobi/ml-examples
Posted by am...@apache.org.
Merge pull request #165 from InMobi/ml-examples
MLTask workflow for combined training & evaluation
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/90dfa2a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/90dfa2a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/90dfa2a4
Branch: refs/heads/toapache
Commit: 90dfa2a4ee6b37694acaf0d3740e8930677fa4e2
Parents: dbe2c08 c2a704c
Author: Jaideep Dhok <ja...@gmail.com>
Authored: Wed Nov 12 14:16:19 2014 +0530
Committer: Jaideep Dhok <ja...@gmail.com>
Committed: Wed Nov 12 14:16:19 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/client/LensConnection.java | 20 +-
.../data/naive_bayes/naive_bayes_train.data | 6 +
lens-ml-lib/pom.xml | 6 +-
.../org/apache/lens/client/LensMLClient.java | 50 ++-
.../apache/lens/client/LensMLJerseyClient.java | 57 +++-
.../java/org/apache/lens/ml/ExampleUtils.java | 45 ++-
.../main/java/org/apache/lens/ml/LensML.java | 6 +-
.../java/org/apache/lens/ml/LensMLImpl.java | 43 ++-
.../main/java/org/apache/lens/ml/MLModel.java | 2 +
.../java/org/apache/lens/ml/MLTestReport.java | 2 +
.../main/java/org/apache/lens/ml/MLUtils.java | 27 ++
.../org/apache/lens/ml/TableTestingSpec.java | 76 ++++-
.../java/org/apache/lens/ml/dao/MLDBUtils.java | 105 +++++++
.../lens/ml/spark/ColumnFeatureFunction.java | 2 +-
.../apache/lens/ml/spark/DoubleValueMapper.java | 6 +-
.../org/apache/lens/ml/spark/HiveTableRDD.java | 9 +-
.../apache/lens/ml/spark/TableTrainingSpec.java | 49 ++-
.../ml/spark/trainers/BaseSparkTrainer.java | 9 +-
.../java/org/apache/lens/ml/task/MLTask.java | 313 +++++++++++++++++++
.../java/org/apache/lens/server/ml/MLApp.java | 25 +-
.../apache/lens/server/ml/MLServiceImpl.java | 9 +-
.../lens/server/ml/MLServiceResource.java | 28 +-
.../java/org/apache/lens/ml/TestMLResource.java | 151 ++++++++-
lens-ml-lib/testutils/hadoop | 110 +++++++
lens-server/src/test/resources/lens-site.xml | 1 -
25 files changed, 1028 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
[42/50] incubator-lens git commit: Merge pull request #177 from
InMobi/classloader
Posted by am...@apache.org.
Merge pull request #177 from InMobi/classloader
Set session class loader for query conf and rewriter's query conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/351f3dce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/351f3dce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/351f3dce
Branch: refs/heads/toapache
Commit: 351f3dcef263e22bd670b12560b06137589c2278
Parents: 05a481a 220ce56
Author: Amareshwari <am...@inmobi.com>
Authored: Thu Nov 13 14:14:51 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Thu Nov 13 14:14:51 2014 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/driver/cube/RewriteUtil.java | 1 +
lens-server/src/main/java/org/apache/lens/server/LensService.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
[31/50] incubator-lens git commit: 1. Removed setter for session
handle in LensConnection 2. Removed unnecessary config from lens-server's
test site.xml
Posted by am...@apache.org.
1. Removed setter for session handle in LensConnection
2. Removed unnecessary config from lens-server's test site.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/c2a704c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/c2a704c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/c2a704c1
Branch: refs/heads/toapache
Commit: c2a704c1e6b246a068dbf1627f576248ba9435a1
Parents: acb2618
Author: jdhok <ja...@inmobi.com>
Authored: Wed Nov 12 12:57:07 2014 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Wed Nov 12 12:57:07 2014 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/client/LensConnection.java | 12 +++++++++++-
.../main/java/org/apache/lens/client/LensMLClient.java | 2 +-
.../org/apache/lens/client/LensMLJerseyClient.java | 1 -
lens-server/src/test/resources/lens-site.xml | 13 -------------
4 files changed, 12 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c2a704c1/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
index bf63ee7..bc07e68 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
@@ -60,7 +60,6 @@ public class LensConnection {
/** The session handle. */
@Getter
- @Setter
private LensSessionHandle sessionHandle;
/**
@@ -74,6 +73,17 @@ public class LensConnection {
}
/**
+ * Construct a connection to lens server specified by connection parameters with an already established session
+ *
+ * @param params
+ * parameters to be used for creating a connection
+ */
+ public LensConnection(LensConnectionParams params, LensSessionHandle sessionHandle) {
+ this.params = params;
+ this.sessionHandle = sessionHandle;
+ }
+
+ /**
* Check if the connection is opened. Please note that,lens connections are persistent connections. But a session
* mapped by ID running on the lens server.
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c2a704c1/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
index f880ead..5c5bc65 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
@@ -71,7 +71,7 @@ public class LensMLClient implements LensML, Closeable {
* the client conf
*/
public LensMLClient(LensConnectionParams clientConf, LensSessionHandle sessionHandle) {
- client = new LensMLJerseyClient(new LensConnection(clientConf), sessionHandle);
+ client = new LensMLJerseyClient(new LensConnection(clientConf, sessionHandle), sessionHandle);
LOG.info("Client created with existing session");
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c2a704c1/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
index 3e41f52..0cc1824 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
@@ -80,7 +80,6 @@ public class LensMLJerseyClient {
public LensMLJerseyClient(LensConnection connection, LensSessionHandle sessionHandle) {
this.connection = connection;
this.sessionHandle = sessionHandle;
- connection.setSessionHandle(sessionHandle);
}
protected WebTarget getMLWebTarget() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c2a704c1/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index 1dfb0b6..ebf4ddb 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -136,17 +136,4 @@
<value>org.apache.lens.server.MockNonLensService</value>
<description>Implementation class for session service</description>
</property>
-
-<property>
- <name>lens.server.ws.resourcenames</name>
- <value>session,metastore,query,quota,scheduler,index,ml</value>
- <description>These JAX-RS resources would be started in the specified order when lens-server starts up</description>
-</property>
-
-<property>
- <name>lens.server.ml.ws.resource.impl</name>
- <value>org.apache.lens.server.ml.MLServiceResource</value>
- <description>Implementation class for Session Resource</description>
-</property>
-
</configuration>
[43/50] incubator-lens git commit: Move to released version of hive
Posted by am...@apache.org.
Move to released version of hive
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/0f46c24c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/0f46c24c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/0f46c24c
Branch: refs/heads/toapache
Commit: 0f46c24ce6ed7be14c7c8d3b737fe3ad6fb1ee21
Parents: 351f3dc
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 13 14:17:48 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 13 14:17:48 2014 +0530
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0f46c24c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 936b760..8158239 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,7 +33,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<powermock.version>1.4.12</powermock.version>
- <hive.version>0.13.3-inm-SNAPSHOT</hive.version>
+ <hive.version>0.13.3-inm</hive.version>
<jetty.version>6.1.26</jetty.version>
<jersey.version>2.3.1</jersey.version>
<grizzly.version>2.3.6</grizzly.version>
[19/50] incubator-lens git commit: Fix NPE in stop() when init() fails
Posted by am...@apache.org.
Fix NPE in stop() when init() fails
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/e71126cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/e71126cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/e71126cf
Branch: refs/heads/toapache
Commit: e71126cff02347d7926ca59a6555302893343a51
Parents: 480457a
Author: jdhok <ja...@inmobi.com>
Authored: Mon Nov 10 11:17:42 2014 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Mon Nov 10 11:17:42 2014 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/server/LensServices.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e71126cf/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index cf3b21c..f345b72 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -46,7 +46,7 @@ import org.apache.lens.server.stats.StatisticsService;
import org.apache.lens.server.user.UserConfigLoaderFactory;
/**
- * The Class LensServices.
+ * Manage lifecycle of all Lens services
*/
public class LensServices extends CompositeService implements ServiceProvider {
@@ -313,7 +313,11 @@ public class LensServices extends CompositeService implements ServiceProvider {
for (LensService service : lensServices) {
service.prepareStopping();
}
- timer.cancel();
+
+ if (timer != null) {
+ timer.cancel();
+ }
+
try {
// persist all the services
persistLensServiceState();
[50/50] incubator-lens git commit: Update the git repo link
Posted by am...@apache.org.
Update the git repo link
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d34a8f7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d34a8f7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d34a8f7b
Branch: refs/heads/toapache
Commit: d34a8f7b92c40a34b41b888d51517b84a6264f30
Parents: a47ecd5
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 13 17:13:35 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 13 17:13:35 2014 +0530
----------------------------------------------------------------------
lens-server/enunciate.xml | 2 +-
src/site/apt/developer/contribute.apt | 2 +-
src/site/apt/gettingstarted/install-steps.apt | 2 +-
tools/scripts/generate-site-public.sh | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d34a8f7b/lens-server/enunciate.xml
----------------------------------------------------------------------
diff --git a/lens-server/enunciate.xml b/lens-server/enunciate.xml
index b4fafae..de134d3 100644
--- a/lens-server/enunciate.xml
+++ b/lens-server/enunciate.xml
@@ -22,7 +22,7 @@
<enunciate label="lens" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="http://enunciate.codehaus.org/schemas/enunciate-1.25.xsd">
<namespaces>
- <namespace uri="https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git" id="lens"/>
+ <namespace uri="https://git-wip-us.apache.org/repos/asf/incubator-lens.git" id="lens"/>
</namespaces>
<services>
<rest defaultRestSubcontext="/">
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d34a8f7b/src/site/apt/developer/contribute.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/developer/contribute.apt b/src/site/apt/developer/contribute.apt
index dcf033c..c196d66 100644
--- a/src/site/apt/developer/contribute.apt
+++ b/src/site/apt/developer/contribute.apt
@@ -29,7 +29,7 @@ Developer Documentation : How to contribute to Apache Lens?
** Source Repository
Lens uses {{{http://git-scm.com/} git}} for its code repository. The repository is available at
- {{{https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git} https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git}}.
+ {{{https://git-wip-us.apache.org/repos/asf/incubator-lens.git} https://git-wip-us.apache.org/repos/asf/incubator-lens.git}}.
** Build tools
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d34a8f7b/src/site/apt/gettingstarted/install-steps.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/gettingstarted/install-steps.apt b/src/site/apt/gettingstarted/install-steps.apt
index 9560e0c..bd19c0d 100644
--- a/src/site/apt/gettingstarted/install-steps.apt
+++ b/src/site/apt/gettingstarted/install-steps.apt
@@ -27,7 +27,7 @@ Installing and running lens
+---+
- git clone https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git
+ git clone https://git-wip-us.apache.org/repos/asf/incubator-lens.git
cd lens
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d34a8f7b/tools/scripts/generate-site-public.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/generate-site-public.sh b/tools/scripts/generate-site-public.sh
index afe956b..29f4b87 100755
--- a/tools/scripts/generate-site-public.sh
+++ b/tools/scripts/generate-site-public.sh
@@ -24,7 +24,7 @@ die() {
exit 1
}
-REPO=https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git
+REPO=https://git-wip-us.apache.org/repos/asf/incubator-lens.git
TMP=/tmp/lens-site-stage
STAGE=`pwd`/target/staging
REST_DIR=`pwd`/lens-server/target/site/wsdocs
[27/50] incubator-lens git commit: documentation fix
Posted by am...@apache.org.
documentation fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/bbe14ef6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/bbe14ef6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/bbe14ef6
Branch: refs/heads/toapache
Commit: bbe14ef65a06f0c99c7b4a67e51ac1c7cf0b1f8f
Parents: fab4c56
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Tue Nov 11 15:57:53 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Tue Nov 11 15:57:53 2014 +0530
----------------------------------------------------------------------
lens-server/src/main/resources/lensserver-default.xml | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bbe14ef6/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index 38f74a0..986c714 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -439,10 +439,13 @@
<value>/path/to/propertyfile</value>
<description>Required for PROPERTYBASED user resolver.
when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to
- determine cluster user. example schema of the file is:
- user1, user2, user3 = clusteruser1
- user4, user5 = clusteruser2
- * = clusteruser3
+ determine cluster user. Each line should contain username followed by DOT followed by property full name
+ followed by equal-to sign and followed by value.
+ example schema of the file is:
+ user1.lens.server.cluster.user=clusteruser1
+ user1.mapred.job.queue.name=queue1
+ *.lens.server.cluster.user=defaultclusteruser
+ *.mapred.job.queue.name=default
</description>
</property>
<property>
[10/50] incubator-lens git commit: Merge branch 'develop' into
ml-examples
Posted by am...@apache.org.
Merge branch 'develop' into ml-examples
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/a97d0d05
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/a97d0d05
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/a97d0d05
Branch: refs/heads/toapache
Commit: a97d0d0534181287b2d7ff3caa61a65779421f1c
Parents: 88b900c 01a5c35
Author: jdhok <ja...@inmobi.com>
Authored: Fri Nov 7 12:52:18 2014 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Fri Nov 7 12:52:18 2014 +0530
----------------------------------------------------------------------
.gitignore | 2 +-
.../lens/cli/commands/LensQueryCommands.java | 2 +-
.../lens/cube/metadata/AbstractBaseTable.java | 8 +-
.../org/apache/lens/cube/metadata/Cube.java | 23 ++-
.../apache/lens/cube/metadata/Dimension.java | 8 +-
.../lens/cube/metadata/MetastoreConstants.java | 1 +
.../lens/cube/metadata/MetastoreUtil.java | 51 ++++++
.../lens/cube/parse/CubeQueryConfUtil.java | 2 -
.../apache/lens/driver/cube/RewriteUtil.java | 5 +-
.../lens/cube/metadata/TestMetastoreUtil.java | 61 +++++++
.../apache/lens/cube/parse/CubeTestSetup.java | 25 ++-
.../lens/cube/parse/TestQueryRewrite.java | 2 +-
lens-driver-hive/pom.xml | 3 +
.../org/apache/lens/driver/hive/HiveDriver.java | 6 +-
.../apache/lens/driver/hive/HiveQueryPlan.java | 65 ++++----
.../apache/lens/driver/hive/TestHiveDriver.java | 167 ++++++++++++++-----
.../lens/driver/hive/TestRemoteHiveDriver.java | 71 ++------
lens-driver-hive/testng.xml | 8 +-
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 27 ++-
.../driver/jdbc/TestColumnarSQLRewriter.java | 31 ++--
.../src/main/resources/dim3-storage-tables.xml | 2 +-
.../src/main/resources/dim4-storage-tables.xml | 2 +-
.../main/resources/local-cluster-storage.xml | 2 +-
lens-ml-lib/src/test/resources/lens-site.xml | 4 +-
.../server/api/events/LensEventService.java | 8 -
.../apache/lens/server/EventServiceImpl.java | 54 ++----
.../apache/lens/server/query/LensServerDAO.java | 2 +-
.../server/query/QueryExecutionServiceImpl.java | 2 +-
.../apache/lens/server/TestServiceProvider.java | 1 +
.../lens/server/query/TestEventService.java | 66 ++++++--
.../apache/lens/server/query/TestLensDAO.java | 1 +
.../lens/server/query/TestQueryService.java | 6 +-
.../server/stats/TestLogStatisticsStore.java | 1 +
.../stats/TestStatisticsLogFileScannerTask.java | 1 +
.../TestStatisticsLogPartitionHandler.java | 1 +
.../lens/server/user/TestUserConfigLoader.java | 19 ++-
.../org/apache/lens/storage/db/DBStorage.java | 14 +-
pom.xml | 1 +
tools/conf/client/lens-client-site.xml | 10 +-
tools/conf/server/hivedriver-site.xml | 2 +-
tools/conf/server/jdbcdriver-site.xml | 2 +-
41 files changed, 475 insertions(+), 294 deletions(-)
----------------------------------------------------------------------
[07/50] incubator-lens git commit: Merge pull request #164 from
InMobi/different-db
Posted by am...@apache.org.
Merge pull request #164 from InMobi/different-db
Fix executing queries in a different db than default
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/2762a5b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/2762a5b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/2762a5b1
Branch: refs/heads/toapache
Commit: 2762a5b1458519953762d899bb5937a6d821ffc4
Parents: 76c828f 79a9a37
Author: Amareshwari <am...@inmobi.com>
Authored: Fri Nov 7 10:17:37 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Fri Nov 7 10:17:37 2014 +0530
----------------------------------------------------------------------
.../apache/lens/cube/parse/CubeTestSetup.java | 4 +-
.../lens/cube/parse/TestQueryRewrite.java | 2 +-
lens-driver-hive/pom.xml | 3 +
.../org/apache/lens/driver/hive/HiveDriver.java | 6 +-
.../apache/lens/driver/hive/HiveQueryPlan.java | 65 ++++----
.../apache/lens/driver/hive/TestHiveDriver.java | 167 ++++++++++++++-----
.../lens/driver/hive/TestRemoteHiveDriver.java | 71 ++------
lens-driver-hive/testng.xml | 8 +-
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 27 ++-
.../driver/jdbc/TestColumnarSQLRewriter.java | 31 ++--
10 files changed, 222 insertions(+), 162 deletions(-)
----------------------------------------------------------------------
[30/50] incubator-lens git commit: Merge pull request #174 from
prongs/cube_measure_attrs
Posted by am...@apache.org.
Merge pull request #174 from prongs/cube_measure_attrs
adding min and max for cube measures
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/dbe2c084
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/dbe2c084
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/dbe2c084
Branch: refs/heads/toapache
Commit: dbe2c08473ba0c3c8a86b20d218416c3ecd6d2b2
Parents: bbd6a1c aea791f
Author: Amareshwari <am...@inmobi.com>
Authored: Tue Nov 11 16:32:05 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Tue Nov 11 16:32:05 2014 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 2 ++
.../lens/cube/metadata/ColumnMeasure.java | 13 ++++++++----
.../apache/lens/cube/metadata/CubeMeasure.java | 22 +++++++++++++++++++-
.../lens/cube/metadata/MetastoreConstants.java | 2 ++
.../lens/cube/metadata/MetastoreUtil.java | 8 +++++++
.../cube/metadata/TestCubeMetastoreClient.java | 6 +++---
.../apache/lens/server/metastore/JAXBUtils.java | 4 +++-
7 files changed, 48 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[44/50] incubator-lens git commit: Update config doc
Posted by am...@apache.org.
Update config doc
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b5c5a923
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b5c5a923
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b5c5a923
Branch: refs/heads/toapache
Commit: b5c5a923161e0715bbb5f2c5dbb8884b16f4d390
Parents: 0f46c24
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 13 14:54:19 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 13 14:54:19 2014 +0530
----------------------------------------------------------------------
src/site/apt/admin/config.apt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b5c5a923/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index abaf2dc..e982b07 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -174,7 +174,7 @@ Lens server configuration
*--+--+---+--+
|75|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|76|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. example schema of the file is: user1, user2, user3 = clusteruser1 user4, user5 = clusteruser2 * = clusteruser3 |
+|76|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default |
*--+--+---+--+
|77|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
*--+--+---+--+
[11/50] incubator-lens git commit: Merge branch 'develop' of
github.com:InMobi/grill into develop
Posted by am...@apache.org.
Merge branch 'develop' of github.com:InMobi/grill into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/5bce5c4d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/5bce5c4d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/5bce5c4d
Branch: refs/heads/toapache
Commit: 5bce5c4d479b188371f0e37cba1a5a2b5a8f606b
Parents: 699d4c8 01a5c35
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Fri Nov 7 13:02:17 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Fri Nov 7 13:02:17 2014 +0530
----------------------------------------------------------------------
.../apache/lens/cube/parse/CubeTestSetup.java | 4 +-
.../lens/cube/parse/TestQueryRewrite.java | 2 +-
lens-driver-hive/pom.xml | 3 +
.../org/apache/lens/driver/hive/HiveDriver.java | 6 +-
.../apache/lens/driver/hive/HiveQueryPlan.java | 65 ++++----
.../apache/lens/driver/hive/TestHiveDriver.java | 167 ++++++++++++++-----
.../lens/driver/hive/TestRemoteHiveDriver.java | 71 ++------
lens-driver-hive/testng.xml | 8 +-
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 27 ++-
.../driver/jdbc/TestColumnarSQLRewriter.java | 31 ++--
.../lens/server/query/TestQueryService.java | 6 +-
11 files changed, 225 insertions(+), 165 deletions(-)
----------------------------------------------------------------------
[05/50] incubator-lens git commit: Cleanup db in TestQueryRewrite
properly
Posted by am...@apache.org.
Cleanup db in TestQueryRewrite properly
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/e1ee94ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/e1ee94ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/e1ee94ee
Branch: refs/heads/toapache
Commit: e1ee94ee6fce29bcd7dad98766e0b2f9a2fc9752
Parents: 09673fe
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 6 17:50:02 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 6 17:50:02 2014 +0530
----------------------------------------------------------------------
.../src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java | 4 ++--
.../test/java/org/apache/lens/cube/parse/TestQueryRewrite.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e1ee94ee/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 173f8e0..5539787 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -1506,9 +1506,9 @@ public class CubeTestSetup {
}
}
- public void dropSources(HiveConf conf) throws Exception {
+ public void dropSources(HiveConf conf, String dbName) throws Exception {
Hive metastore = Hive.get(conf);
- metastore.dropDatabase(SessionState.get().getCurrentDatabase(), true, true, true);
+ metastore.dropDatabase(dbName, true, true, true);
}
private void createCubeFactsWithValidColumns(CubeMetastoreClient client) throws HiveException {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e1ee94ee/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
index c4b6600..492ce68 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
@@ -42,7 +42,7 @@ public abstract class TestQueryRewrite {
@AfterSuite
public static void tearDown() throws Exception {
- setup.dropSources(hconf);
+ setup.dropSources(hconf, TestQueryRewrite.class.getSimpleName());
}
@BeforeClass
[23/50] incubator-lens git commit: Fixed errors when shutting down
server in case of failed init
Posted by am...@apache.org.
Fixed errors when shutting down server in case of failed init
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b690a923
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b690a923
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b690a923
Branch: refs/heads/toapache
Commit: b690a923612e9e2fd58e909ad4f22ded164841f3
Parents: fab4c56
Author: jdhok <ja...@inmobi.com>
Authored: Tue Nov 11 12:21:09 2014 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Tue Nov 11 12:21:09 2014 +0530
----------------------------------------------------------------------
.../apache/lens/server/EventServiceImpl.java | 20 +++----
.../org/apache/lens/server/LensServices.java | 34 ++++++------
.../apache/lens/server/MetricsServiceImpl.java | 17 ++++--
.../lens/server/TestStartupOnMetastoreDown.java | 56 ++++++++++++++++++++
4 files changed, 98 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b690a923/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
index 736b2d5..aadd0e1 100644
--- a/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
@@ -197,19 +197,21 @@ public class EventServiceImpl extends AbstractService implements LensEventServic
*/
@Override
public void stop() {
- List<Runnable> pending = eventHandlerPool.shutdownNow();
- if (pending != null && !pending.isEmpty()) {
- StringBuilder pendingMsg = new StringBuilder("Pending Events:");
- for (Runnable handler : pending) {
- if (handler instanceof EventHandler) {
- pendingMsg.append(((EventHandler) handler).event.getEventId()).append(",");
+ if (eventHandlerPool != null) {
+ List<Runnable> pending = eventHandlerPool.shutdownNow();
+ if (pending != null && !pending.isEmpty()) {
+ StringBuilder pendingMsg = new StringBuilder("Pending Events:");
+ for (Runnable handler : pending) {
+ if (handler instanceof EventHandler) {
+ pendingMsg.append(((EventHandler) handler).event.getEventId()).append(",");
+ }
}
+ LOG.info("Event listener service stopped while " + pending.size() + " events still pending");
+ LOG.info(pendingMsg.toString());
}
- LOG.info("Event listener service stopped while " + pending.size() + " events still pending");
- LOG.info(pendingMsg.toString());
}
- super.stop();
LOG.info("Event service stopped");
+ super.stop();
}
public Map<Class<? extends LensEvent>, List<LensEventListener>> getEventListeners() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b690a923/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index f345b72..fe2fc75 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -266,24 +266,26 @@ public class LensServices extends CompositeService implements ServiceProvider {
*/
private synchronized void persistLensServiceState() throws IOException {
if (conf.getBoolean(LensConfConstants.SERVER_RESTART_ENABLED, LensConfConstants.DEFAULT_SERVER_RESTART_ENABLED)) {
- FileSystem fs = persistDir.getFileSystem(conf);
- LOG.info("Persisting server state in " + persistDir);
-
- for (LensService service : lensServices) {
- LOG.info("Persisting state of service:" + service.getName());
- Path serviceWritePath = new Path(persistDir, service.getName() + ".out");
- ObjectOutputStream out = null;
- try {
- out = new ObjectOutputStream(fs.create(serviceWritePath));
- service.writeExternal(out);
- } finally {
- if (out != null) {
- out.close();
+ if (persistDir != null) {
+ FileSystem fs = persistDir.getFileSystem(conf);
+ LOG.info("Persisting server state in " + persistDir);
+
+ for (LensService service : lensServices) {
+ LOG.info("Persisting state of service:" + service.getName());
+ Path serviceWritePath = new Path(persistDir, service.getName() + ".out");
+ ObjectOutputStream out = null;
+ try {
+ out = new ObjectOutputStream(fs.create(serviceWritePath));
+ service.writeExternal(out);
+ } finally {
+ if (out != null) {
+ out.close();
+ }
}
+ Path servicePath = getServicePersistPath(service);
+ fs.rename(serviceWritePath, servicePath);
+ LOG.info("Persisted service " + service.getName() + " to " + servicePath);
}
- Path servicePath = getServicePersistPath(service);
- fs.rename(serviceWritePath, servicePath);
- LOG.info("Persisted service " + service.getName() + " to " + servicePath);
}
} else {
LOG.info("Server restart is not enabled. Not persisting the server state");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b690a923/lens-server/src/main/java/org/apache/lens/server/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/MetricsServiceImpl.java
index 01cbb16..b0774e0 100644
--- a/lens-server/src/main/java/org/apache/lens/server/MetricsServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/MetricsServiceImpl.java
@@ -279,11 +279,20 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
public synchronized void stop() {
// unregister
LensEventService eventService = (LensEventService) LensServices.get().getService(LensEventService.NAME);
- eventService.removeListener(queryStatusListener);
- queryStatusListener.stop();
- for (ScheduledReporter reporter : reporters) {
- reporter.stop();
+ if (eventService != null) {
+ eventService.removeListener(queryStatusListener);
+ }
+
+ if (queryStatusListener != null) {
+ queryStatusListener.stop();
}
+
+ if (reporters != null) {
+ for (ScheduledReporter reporter : reporters) {
+ reporter.stop();
+ }
+ }
+
LOG.info("Stopped metrics service");
super.stop();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b690a923/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
new file mode 100644
index 0000000..2254bd8
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestStartupOnMetastoreDown {
+ private static Log LOG = LogFactory.getLog(TestStartupOnMetastoreDown.class);
+
+ // @Test
+ public void testServicesStartOnMetastoreDown() throws Exception {
+ LensServices services = new LensServices(LensServices.LENS_SERVICES_NAME);
+ HiveConf hiveConf = new HiveConf();
+
+ // Set metastore uri to an invalid location
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:49153");
+
+ try {
+ services.init(hiveConf);
+ Assert.fail("Expected init to fail because of invalid metastore config");
+ } catch (Throwable th) {
+ Assert.assertTrue(th.getMessage().contains(
+ "Unable to instantiate org.apache.hadoop.hive.metastore.HiveMetaStoreClient"));
+ } finally {
+ try {
+ services.stop();
+ } catch (Exception exc) {
+ LOG.error("Error stopping services", exc);
+ Assert.fail("services.stop() got unexpected exception " + exc);
+ }
+ Hive.closeCurrent();
+ }
+ }
+
+}
[28/50] incubator-lens git commit: Merge pull request #175 from
prongs/doc_fix
Posted by am...@apache.org.
Merge pull request #175 from prongs/doc_fix
documentation fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/bbd6a1c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/bbd6a1c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/bbd6a1c0
Branch: refs/heads/toapache
Commit: bbd6a1c0aa489ebe01f6f780ea812380aabe66b9
Parents: e21f750 bbe14ef
Author: Amareshwari <am...@inmobi.com>
Authored: Tue Nov 11 16:05:22 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Tue Nov 11 16:05:22 2014 +0530
----------------------------------------------------------------------
lens-server/src/main/resources/lensserver-default.xml | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[06/50] incubator-lens git commit: Fix test failures and explain plan
parsing in hive
Posted by am...@apache.org.
Fix test failures and explain plan parsing in hive
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/79a9a373
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/79a9a373
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/79a9a373
Branch: refs/heads/toapache
Commit: 79a9a37398278027576b8074e17321089982290a
Parents: e1ee94e
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 6 17:51:07 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 6 17:51:07 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 6 +-
.../apache/lens/driver/hive/HiveQueryPlan.java | 63 +++----
.../apache/lens/driver/hive/TestHiveDriver.java | 167 ++++++++++++++-----
.../lens/driver/hive/TestRemoteHiveDriver.java | 71 ++------
.../driver/jdbc/TestColumnarSQLRewriter.java | 25 +--
5 files changed, 193 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/79a9a373/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 84341fb..5ea20b2 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -94,7 +94,8 @@ public class HiveDriver implements LensDriver {
// connections need to be separate for each user and each thread
/** The thread connections. */
- private final Map<String, Map<Long, ExpirableConnection>> threadConnections = new HashMap<String, Map<Long, ExpirableConnection>>();
+ private final Map<String, Map<Long, ExpirableConnection>> threadConnections =
+ new HashMap<String, Map<Long, ExpirableConnection>>();
/** The thrift conn expiry queue. */
private final DelayQueue<ExpirableConnection> thriftConnExpiryQueue = new DelayQueue<ExpirableConnection>();
@@ -274,7 +275,6 @@ public class HiveDriver implements LensDriver {
@Override
public void configure(Configuration conf) throws LensException {
this.driverConf = new HiveConf(conf, HiveDriver.class);
- ;
this.driverConf.addResource("hivedriver-default.xml");
this.driverConf.addResource("hivedriver-site.xml");
connectionClass = this.driverConf.getClass(HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class,
@@ -294,7 +294,7 @@ public class HiveDriver implements LensDriver {
Configuration explainConf = new Configuration(conf);
explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
String explainQuery = "EXPLAIN EXTENDED " + query;
- QueryContext explainQueryCtx = new QueryContext(explainQuery, null, explainConf);
+ QueryContext explainQueryCtx = new QueryContext(explainQuery, SessionState.get().getUserName(), explainConf);
// Get result set of explain
HiveInMemoryResultSet inMemoryResultSet = (HiveInMemoryResultSet) execute(explainQueryCtx);
List<String> explainOutput = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/79a9a373/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
index ac8cbee..78f776a 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
@@ -143,18 +143,7 @@ public class HiveQueryPlan extends DriverQueryPlan {
}
break;
case TABLE_SCAN:
- if (tr.startsWith("name:")) {
- String tableName = tr.replace("name:", "").trim();
- tablesQueried.add(tableName);
- Table tbl = metastore.getTable(tableName);
- String costStr = tbl.getParameters().get(LensConfConstants.STORAGE_COST);
-
- Double weight = 1d;
- if (costStr != null) {
- weight = Double.parseDouble(costStr);
- }
- tableWeights.put(tableName, weight);
- }
+ // no op
break;
case JOIN:
if (tr.equals("condition map:")) {
@@ -177,37 +166,49 @@ public class HiveQueryPlan extends DriverQueryPlan {
}
break;
case PARTITION:
- if (tr.equals("partition values:")) {
- i++;
- List<String> partVals = new ArrayList<String>();
- // Look ahead until we reach partition properties
- String lineAhead = null;
- for (; i < explainOutput.size(); i++) {
- if (explainOutput.get(i).trim().equals("properties:")) {
- break;
+ String partConditionStr = null;
+ for (; i < explainOutput.size(); i++) {
+ if (explainOutput.get(i).trim().equals("partition values:")) {
+ List<String> partVals = new ArrayList<String>();
+ // Look ahead until we reach partition properties
+ String lineAhead = null;
+ for (; i < explainOutput.size(); i++) {
+ if (explainOutput.get(i).trim().equals("properties:")) {
+ break;
+ }
+ lineAhead = explainOutput.get(i).trim();
+ partVals.add(lineAhead);
}
- lineAhead = explainOutput.get(i).trim();
- partVals.add(lineAhead);
- }
-
- String partConditionStr = StringUtils.join(partVals, ";");
+ partConditionStr = StringUtils.join(partVals, ";");
+ }
// Now seek table name
- for (; i < explainOutput.size(); i++) {
- if (explainOutput.get(i).trim().startsWith("name:")) {
- String table = explainOutput.get(i).trim().substring("name:".length()).trim();
+ if (explainOutput.get(i).trim().startsWith("name:")) {
+ String table = explainOutput.get(i).trim().substring("name:".length()).trim();
+ // update tables queried and weights
+ if (!tablesQueried.contains(table)) {
+ tablesQueried.add(table);
+ Table tbl = metastore.getTable(table);
+ String costStr = tbl.getParameters().get(LensConfConstants.STORAGE_COST);
+
+ Double weight = 1d;
+ if (costStr != null) {
+ weight = Double.parseDouble(costStr);
+ }
+ tableWeights.put(table, weight);
+ }
+
+ if (partConditionStr != null) {
List<String> tablePartitions = partitions.get(table);
if (tablePartitions == null) {
tablePartitions = new ArrayList<String>();
partitions.put(table, tablePartitions);
}
tablePartitions.add(partConditionStr);
-
- break;
}
+ break;
}
}
-
break;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/79a9a373/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index f5312a0..45cebde 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -25,6 +25,7 @@ import java.util.*;
import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -51,7 +52,10 @@ import org.apache.lens.server.api.driver.LensResultSetMetadata;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
-import org.testng.annotations.*;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
/**
* The Class TestHiveDriver.
@@ -65,13 +69,16 @@ public class TestHiveDriver {
public final String TEST_OUTPUT_DIR = "target/" + this.getClass().getSimpleName() + "/test-output";
/** The conf. */
- protected HiveConf conf = new HiveConf();
+ protected HiveConf conf;
/** The driver. */
protected HiveDriver driver;
/** The data base. */
- public final String DATA_BASE = this.getClass().getSimpleName();
+ public String DATA_BASE = this.getClass().getSimpleName().toLowerCase();
+
+ protected String sessionid;
+ protected SessionState ss;
/**
* Before test.
@@ -84,27 +91,50 @@ public class TestHiveDriver {
// Check if hadoop property set
System.out.println("###HADOOP_PATH " + System.getProperty("hadoop.bin.path"));
assertNotNull(System.getProperty("hadoop.bin.path"));
- conf.addResource("hivedriver-site.xml");
- conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
- conf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
- SessionState ss = new SessionState(conf, "testuser");
+
+ createDriver();
+ ss = new SessionState(conf, "testuser");
SessionState.start(ss);
Hive client = Hive.get(conf);
Database database = new Database();
- database.setName(TestHiveDriver.class.getSimpleName());
+ database.setName(DATA_BASE);
client.createDatabase(database, true);
- SessionState.get().setCurrentDatabase(TestHiveDriver.class.getSimpleName());
+ SessionState.get().setCurrentDatabase(DATA_BASE);
+ sessionid = SessionState.get().getSessionId();
- driver = new HiveDriver();
- driver.configure(conf);
conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
- QueryContext context = new QueryContext("USE " + TestHiveDriver.class.getSimpleName(), null, conf);
+ QueryContext context = createContext("USE " + DATA_BASE, conf);
driver.execute(context);
conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
- System.out.println("Driver created");
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ }
+
+ protected void createDriver() throws LensException {
+ conf = new HiveConf();
+ conf.addResource("hivedriver-site.xml");
+ conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
+ conf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
+ driver = new HiveDriver();
+ driver.configure(conf);
+ System.out.println("TestHiveDriver created");
+ }
+
+ @BeforeMethod
+ public void setDB() {
+ SessionState.get().setCurrentDatabase(DATA_BASE);
+ }
+
+ protected QueryContext createContext(String query, Configuration conf) {
+ QueryContext context = new QueryContext(query, "testuser", conf);
+ context.setLensSessionIdentifier(sessionid);
+ return context;
+ }
+
+ protected QueryContext createContext(PreparedQueryContext query, Configuration conf) {
+ QueryContext context = new QueryContext(query, "testuser", conf);
+ context.setLensSessionIdentifier(sessionid);
+ return context;
}
/**
@@ -116,7 +146,7 @@ public class TestHiveDriver {
@AfterTest
public void afterTest() throws Exception {
driver.close();
- Hive.get(conf).dropDatabase(TestHiveDriver.class.getSimpleName(), true, true, true);
+ Hive.get(conf).dropDatabase(DATA_BASE, true, true, true);
}
/**
@@ -133,18 +163,45 @@ public class TestHiveDriver {
+ LensConfConstants.STORAGE_COST + "'='500')";
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
// Craete again
- QueryContext context = new QueryContext(createTable, null, conf);
+ QueryContext context = createContext(createTable, conf);
LensResultSet resultSet = driver.execute(context);
assertNull(resultSet);
// Load some data into the table
String dataLoad = "LOAD DATA LOCAL INPATH '" + TEST_DATA_FILE + "' OVERWRITE INTO TABLE " + tableName;
- context = new QueryContext(dataLoad, null, conf);
+ context = createContext(dataLoad, conf);
resultSet = driver.execute(context);
assertNull(resultSet);
Assert.assertEquals(0, driver.getHiveHandleSize());
}
+ /**
+ * Creates the test table.
+ *
+ * @param tableName
+ * the table name
+ * @throws Exception
+ * the exception
+ */
+ protected void createPartitionedTable(String tableName) throws Exception {
+ System.out.println("Hadoop Location: " + System.getProperty("hadoop.bin.path"));
+ String createTable = "CREATE TABLE IF NOT EXISTS " + tableName + "(ID STRING)" + " PARTITIONED BY (dt string) TBLPROPERTIES ('"
+ + LensConfConstants.STORAGE_COST + "'='500')";
+ conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
+ // Craete again
+ QueryContext context = createContext(createTable, conf);
+ LensResultSet resultSet = driver.execute(context);
+ assertNull(resultSet);
+
+ // Load some data into the table
+ String dataLoad = "LOAD DATA LOCAL INPATH '" + TEST_DATA_FILE + "' OVERWRITE INTO TABLE " + tableName
+ + " partition (dt='today')";
+ context = createContext(dataLoad, conf);
+ resultSet = driver.execute(context);
+ assertNull(resultSet);
+ Assert.assertEquals(0, driver.getHiveHandleSize());
+ }
+
// Tests
/**
* Test insert overwrite conf.
@@ -157,7 +214,7 @@ public class TestHiveDriver {
createTestTable("test_insert_overwrite");
conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
String query = "SELECT ID FROM test_insert_overwrite";
- QueryContext context = new QueryContext(query, null, conf);
+ QueryContext context = createContext(query, conf);
driver.addPersistentPath(context);
assertEquals(context.getUserQuery(), query);
assertNotNull(context.getDriverQuery());
@@ -176,14 +233,14 @@ public class TestHiveDriver {
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
Hive.get(conf).dropTable("test_temp_output");
String query = "CREATE TABLE test_temp_output AS SELECT ID FROM test_temp";
- QueryContext context = new QueryContext(query, null, conf);
+ QueryContext context = createContext(query, conf);
LensResultSet resultSet = driver.execute(context);
assertNull(resultSet);
Assert.assertEquals(0, driver.getHiveHandleSize());
// fetch results from temp table
String select = "SELECT * FROM test_temp_output";
- context = new QueryContext(select, null, conf);
+ context = createContext(select, conf);
resultSet = driver.execute(context);
Assert.assertEquals(0, driver.getHiveHandleSize());
validateInMemoryResult(resultSet, "test_temp_output");
@@ -203,11 +260,11 @@ public class TestHiveDriver {
// Execute a select query
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
String select = "SELECT ID FROM test_execute";
- QueryContext context = new QueryContext(select, null, conf);
+ QueryContext context = createContext(select, conf);
resultSet = driver.execute(context);
validateInMemoryResult(resultSet);
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
- context = new QueryContext(select, null, conf);
+ context = createContext(select, conf);
resultSet = driver.execute(context);
validatePersistentResult(resultSet, TEST_DATA_FILE, context.getHDFSResultDir(), false);
conf.set(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT,
@@ -215,7 +272,7 @@ public class TestHiveDriver {
+ " WITH SERDEPROPERTIES ('serialization.null.format'='-NA-',"
+ " 'field.delim'=',' ) STORED AS TEXTFILE ");
select = "SELECT ID, null, ID FROM test_execute";
- context = new QueryContext(select, null, conf);
+ context = createContext(select, conf);
resultSet = driver.execute(context);
validatePersistentResult(resultSet, TEST_DATA_FILE, context.getHDFSResultDir(), true);
Assert.assertEquals(0, driver.getHiveHandleSize());
@@ -327,7 +384,7 @@ public class TestHiveDriver {
String expectFail = "SELECT ID FROM test_execute_sync";
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
conf.set("hive.exec.driver.run.hooks", FailHook.class.getCanonicalName());
- QueryContext context = new QueryContext(expectFail, null, conf);
+ QueryContext context = createContext(expectFail, conf);
driver.executeAsync(context);
Assert.assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.FAILED, true, false);
@@ -339,7 +396,7 @@ public class TestHiveDriver {
// Async select query
String select = "SELECT ID FROM test_execute_sync";
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
- context = new QueryContext(select, null, conf);
+ context = createContext(select, conf);
driver.executeAsync(context);
Assert.assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, false, false);
@@ -347,7 +404,7 @@ public class TestHiveDriver {
Assert.assertEquals(0, driver.getHiveHandleSize());
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
- context = new QueryContext(select, null, conf);
+ context = createContext(select, conf);
driver.executeAsync(context);
Assert.assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, true, false);
@@ -359,7 +416,7 @@ public class TestHiveDriver {
+ " WITH SERDEPROPERTIES ('serialization.null.format'='-NA-',"
+ " 'field.delim'=',' ) STORED AS TEXTFILE ");
select = "SELECT ID, null, ID FROM test_execute_sync";
- context = new QueryContext(select, null, conf);
+ context = createContext(select, conf);
driver.executeAsync(context);
Assert.assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, true, true);
@@ -433,7 +490,7 @@ public class TestHiveDriver {
public void testCancelAsyncQuery() throws Exception {
createTestTable("test_cancel_async");
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
- QueryContext context = new QueryContext("SELECT ID FROM test_cancel_async", null, conf);
+ QueryContext context = createContext("SELECT ID FROM test_cancel_async", conf);
driver.executeAsync(context);
driver.cancelQuery(context.getQueryHandle());
driver.updateStatus(context);
@@ -525,12 +582,12 @@ public class TestHiveDriver {
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
conf.set(LensConfConstants.RESULT_SET_PARENT_DIR, TEST_OUTPUT_DIR);
- QueryContext ctx = new QueryContext("SELECT ID FROM test_persistent_result_set", null, conf);
+ QueryContext ctx = createContext("SELECT ID FROM test_persistent_result_set", conf);
LensResultSet resultSet = driver.execute(ctx);
validatePersistentResult(resultSet, TEST_DATA_FILE, ctx.getHDFSResultDir(), false);
Assert.assertEquals(0, driver.getHiveHandleSize());
- ctx = new QueryContext("SELECT ID FROM test_persistent_result_set", null, conf);
+ ctx = createContext("SELECT ID FROM test_persistent_result_set", conf);
driver.executeAsync(ctx);
Assert.assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(ctx, DriverQueryState.SUCCESSFUL, true, false);
@@ -541,14 +598,14 @@ public class TestHiveDriver {
"ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'"
+ " WITH SERDEPROPERTIES ('serialization.null.format'='-NA-',"
+ " 'field.delim'=',' ) STORED AS TEXTFILE ");
- ctx = new QueryContext("SELECT ID, null, ID FROM test_persistent_result_set", null, conf);
+ ctx = createContext("SELECT ID, null, ID FROM test_persistent_result_set", conf);
resultSet = driver.execute(ctx);
Assert.assertEquals(0, driver.getHiveHandleSize());
validatePersistentResult(resultSet, TEST_DATA_FILE, ctx.getHDFSResultDir(), true);
driver.closeQuery(ctx.getQueryHandle());
Assert.assertEquals(0, driver.getHiveHandleSize());
- ctx = new QueryContext("SELECT ID, null, ID FROM test_persistent_result_set", null, conf);
+ ctx = createContext("SELECT ID, null, ID FROM test_persistent_result_set", conf);
driver.executeAsync(ctx);
Assert.assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(ctx, DriverQueryState.SUCCESSFUL, true, true);
@@ -592,21 +649,23 @@ public class TestHiveDriver {
@Test
public void testExplain() throws Exception {
createTestTable("test_explain");
+ SessionState.setCurrentSessionState(ss);
DriverQueryPlan plan = driver.explain("SELECT ID FROM test_explain", conf);
assertTrue(plan instanceof HiveQueryPlan);
- assertEquals(plan.getTableWeight("test_explain"), 500.0);
+ assertEquals(plan.getTableWeight(DATA_BASE + ".test_explain"), 500.0);
Assert.assertEquals(0, driver.getHiveHandleSize());
// test execute prepare
PreparedQueryContext pctx = new PreparedQueryContext("SELECT ID FROM test_explain", null, conf);
+ SessionState.setCurrentSessionState(ss);
plan = driver.explainAndPrepare(pctx);
- QueryContext qctx = new QueryContext(pctx, null, conf);
+ QueryContext qctx = createContext(pctx, conf);
LensResultSet result = driver.execute(qctx);
Assert.assertEquals(0, driver.getHiveHandleSize());
validateInMemoryResult(result);
// test execute prepare async
- qctx = new QueryContext(pctx, null, conf);
+ qctx = createContext(pctx, conf);
driver.executeAsync(qctx);
assertNotNull(qctx.getDriverOpHandle());
validateExecuteAsync(qctx, DriverQueryState.SUCCESSFUL, false, false);
@@ -616,14 +675,14 @@ public class TestHiveDriver {
Assert.assertEquals(0, driver.getHiveHandleSize());
// for backward compatibility
- qctx = new QueryContext(pctx, null, conf);
+ qctx = createContext(pctx, conf);
qctx.setQueryHandle(new QueryHandle(pctx.getPrepareHandle().getPrepareHandleId()));
result = driver.execute(qctx);
assertNotNull(qctx.getDriverOpHandle());
Assert.assertEquals(0, driver.getHiveHandleSize());
validateInMemoryResult(result);
// test execute prepare async
- qctx = new QueryContext(pctx, null, conf);
+ qctx = createContext(pctx, conf);
qctx.setQueryHandle(new QueryHandle(pctx.getPrepareHandle().getPrepareHandleId()));
driver.executeAsync(qctx);
Assert.assertEquals(1, driver.getHiveHandleSize());
@@ -635,6 +694,30 @@ public class TestHiveDriver {
}
/**
+ * Test explain partitioned table
+ *
+ * @throws Exception
+ * the exception
+ */
+ @Test
+ public void testExplainPartitionedTable() throws Exception {
+ createPartitionedTable("test_part_table");
+ // acquire
+ SessionState.setCurrentSessionState(ss);
+ DriverQueryPlan plan = driver.explain("SELECT ID FROM test_part_table", conf);
+ Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertTrue(plan instanceof HiveQueryPlan);
+ assertNotNull(plan.getTablesQueried());
+ assertEquals(plan.getTablesQueried().size(), 1);
+ System.out.println("Tables:" + plan.getTablesQueried());
+ assertEquals(plan.getTableWeight(DATA_BASE + ".test_part_table"), 500.0);
+ System.out.println("Parts:" + plan.getPartitions());
+ assertFalse(plan.getPartitions().isEmpty());
+ assertTrue(plan.getPartitions().get(DATA_BASE + ".test_part_table").get(0).contains("today"));
+ assertTrue(plan.getPartitions().get(DATA_BASE + ".test_part_table").get(0).contains("dt"));
+ }
+
+ /**
* Test explain output.
*
* @throws Exception
@@ -645,6 +728,7 @@ public class TestHiveDriver {
createTestTable("explain_test_1");
createTestTable("explain_test_2");
+ SessionState.setCurrentSessionState(ss);
DriverQueryPlan plan = driver.explain("SELECT explain_test_1.ID, count(1) FROM "
+ " explain_test_1 join explain_test_2 on explain_test_1.ID = explain_test_2.ID"
+ " WHERE explain_test_1.ID = 'foo' or explain_test_2.ID = 'bar'" + " GROUP BY explain_test_1.ID", conf);
@@ -654,8 +738,8 @@ public class TestHiveDriver {
assertNotNull(plan.getTablesQueried());
assertEquals(plan.getTablesQueried().size(), 2);
assertNotNull(plan.getTableWeights());
- assertTrue(plan.getTableWeights().containsKey("explain_test_1"));
- assertTrue(plan.getTableWeights().containsKey("explain_test_2"));
+ assertTrue(plan.getTableWeights().containsKey(DATA_BASE + ".explain_test_1"));
+ assertTrue(plan.getTableWeights().containsKey(DATA_BASE + ".explain_test_2"));
assertEquals(plan.getNumJoins(), 1);
assertTrue(plan.getPlan() != null && !plan.getPlan().isEmpty());
driver.closeQuery(plan.getHandle());
@@ -671,6 +755,7 @@ public class TestHiveDriver {
public void testExplainOutputPersistent() throws Exception {
createTestTable("explain_test_1");
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
+ SessionState.setCurrentSessionState(ss);
String query2 = "SELECT DISTINCT ID FROM explain_test_1";
PreparedQueryContext pctx = new PreparedQueryContext(query2, null, conf);
DriverQueryPlan plan2 = driver.explainAndPrepare(pctx);
@@ -678,9 +763,9 @@ public class TestHiveDriver {
Assert.assertEquals(0, driver.getHiveHandleSize());
assertNotNull(plan2.getTablesQueried());
assertEquals(plan2.getTablesQueried().size(), 1);
- assertTrue(plan2.getTableWeights().containsKey("explain_test_1"));
+ assertTrue(plan2.getTableWeights().containsKey(DATA_BASE + ".explain_test_1"));
assertEquals(plan2.getNumSels(), 1);
- QueryContext ctx = new QueryContext(pctx, null, conf);
+ QueryContext ctx = createContext(pctx, conf);
LensResultSet resultSet = driver.execute(ctx);
Assert.assertEquals(0, driver.getHiveHandleSize());
HivePersistentResultSet persistentResultSet = (HivePersistentResultSet) resultSet;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/79a9a373/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index 8f792f6..340e00a 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -30,8 +30,6 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.server.HiveServer2;
import org.apache.lens.api.LensException;
@@ -46,9 +44,7 @@ import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.query.QueryContext;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
-import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
/**
@@ -80,14 +76,6 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
@BeforeClass
public static void setupTest() throws Exception {
createHS2Service();
-
- SessionState ss = new SessionState(remoteConf, "testuser");
- SessionState.start(ss);
- Hive client = Hive.get(remoteConf);
- Database database = new Database();
- database.setName(TestRemoteHiveDriver.class.getSimpleName());
- client.createDatabase(database, true);
- SessionState.get().setCurrentDatabase(TestRemoteHiveDriver.class.getSimpleName());
}
/**
@@ -123,7 +111,6 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
@AfterClass
public static void cleanupTest() throws Exception {
stopHS2Service();
- Hive.get(remoteConf).dropDatabase(TestRemoteHiveDriver.class.getSimpleName(), true, true, true);
}
/**
@@ -140,39 +127,13 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
}
}
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.driver.hive.TestHiveDriver#beforeTest()
- */
- @BeforeMethod
- @Override
- public void beforeTest() throws Exception {
+ protected void createDriver() throws LensException {
+ DATA_BASE = TestRemoteHiveDriver.class.getSimpleName().toLowerCase();
conf = new HiveConf(remoteConf);
conf.addResource("hivedriver-site.xml");
- // Check if hadoop property set
- System.out.println("###HADOOP_PATH " + System.getProperty("hadoop.bin.path"));
- Assert.assertNotNull(System.getProperty("hadoop.bin.path"));
driver = new HiveDriver();
driver.configure(conf);
- conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
- conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
- driver.execute(new QueryContext("USE " + TestRemoteHiveDriver.class.getSimpleName(), null, conf));
- conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
- conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
- Assert.assertEquals(0, driver.getHiveHandleSize());
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.driver.hive.TestHiveDriver#afterTest()
- */
- @AfterMethod
- @Override
- public void afterTest() throws Exception {
- LOG.info("Test finished, closing driver");
- driver.close();
+ System.out.println("TestRemoteHiveDriver created");
}
/**
@@ -190,7 +151,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
thConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
final HiveDriver thrDriver = new HiveDriver();
thrDriver.configure(thConf);
- QueryContext ctx = new QueryContext("USE " + TestRemoteHiveDriver.class.getSimpleName(), null, conf);
+ QueryContext ctx = createContext("USE " + DATA_BASE, conf);
thrDriver.execute(ctx);
// Launch a select query
@@ -203,7 +164,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
for (int q = 0; q < QUERIES; q++) {
final QueryContext qctx;
try {
- qctx = new QueryContext("SELECT * FROM test_multithreads", null, conf);
+ qctx = createContext("SELECT * FROM test_multithreads", conf);
thrDriver.executeAsync(qctx);
} catch (LensException e) {
errCount.incrementAndGet();
@@ -270,7 +231,8 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
@Test
public void testHiveDriverPersistence() throws Exception {
System.out.println("@@@@ start_persistence_test");
- HiveConf driverConf = new HiveConf(conf, TestRemoteHiveDriver.class);
+ HiveConf driverConf = new HiveConf(remoteConf, TestRemoteHiveDriver.class);
+ driverConf.addResource("hivedriver-site.xml");
driverConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
final HiveDriver oldDriver = new HiveDriver();
@@ -278,7 +240,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
driverConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
driverConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
- QueryContext ctx = new QueryContext("USE " + TestRemoteHiveDriver.class.getSimpleName(), null, driverConf);
+ QueryContext ctx = createContext("USE " + DATA_BASE, driverConf);
oldDriver.execute(ctx);
Assert.assertEquals(0, oldDriver.getHiveHandleSize());
@@ -286,20 +248,20 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
// Create some ops with a driver
String createTable = "CREATE TABLE IF NOT EXISTS " + tableName + "(ID STRING)";
- ctx = new QueryContext(createTable, null, driverConf);
+ ctx = createContext(createTable, driverConf);
oldDriver.execute(ctx);
// Load some data into the table
String dataLoad = "LOAD DATA LOCAL INPATH '" + TEST_DATA_FILE + "' OVERWRITE INTO TABLE " + tableName;
- ctx = new QueryContext(dataLoad, null, driverConf);
+ ctx = createContext(dataLoad, driverConf);
oldDriver.execute(ctx);
driverConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
driverConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
// Fire two queries
- QueryContext ctx1 = new QueryContext("SELECT * FROM " + tableName, null, driverConf);
+ QueryContext ctx1 = createContext("SELECT * FROM " + tableName, driverConf);
oldDriver.executeAsync(ctx1);
- QueryContext ctx2 = new QueryContext("SELECT ID FROM " + tableName, null, driverConf);
+ QueryContext ctx2 = createContext("SELECT ID FROM " + tableName, driverConf);
oldDriver.executeAsync(ctx2);
Assert.assertEquals(2, oldDriver.getHiveHandleSize());
@@ -404,8 +366,8 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
- QueryContext ctx = new QueryContext("CREATE EXTERNAL TABLE IF NOT EXISTS " + tableName
- + " (ID STRING) PARTITIONED BY (DT STRING, ET STRING)", null, conf);
+ QueryContext ctx = createContext("CREATE EXTERNAL TABLE IF NOT EXISTS " + tableName
+ + " (ID STRING) PARTITIONED BY (DT STRING, ET STRING)", conf);
driver.execute(ctx);
Assert.assertEquals(0, driver.getHiveHandleSize());
@@ -426,8 +388,8 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
FileUtils.writeLines(data, Arrays.asList("one", "two", "three", "four", "five"));
System.out.println("@@ Adding partition " + i);
- QueryContext partCtx = new QueryContext("ALTER TABLE " + tableName + " ADD IF NOT EXISTS PARTITION (DT='p" + i
- + "', ET='1') LOCATION '" + partDir.getPath() + "'", null, conf);
+ QueryContext partCtx = createContext("ALTER TABLE " + tableName + " ADD IF NOT EXISTS PARTITION (DT='p" + i
+ + "', ET='1') LOCATION '" + partDir.getPath() + "'", conf);
driver.execute(partCtx);
}
}
@@ -450,6 +412,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
+ "WHERE table_1.DT='p0' OR table_1.DT='p1' OR table_1.DT='p2' OR table_1.DT='p3' OR table_1.DT='p4' "
+ "AND table_1.ET='1'";
+ SessionState.setCurrentSessionState(ss);
DriverQueryPlan plan = driver.explain(explainQuery, conf);
Assert.assertEquals(0, driver.getHiveHandleSize());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/79a9a373/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index 3533146..b5088f8 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -22,6 +22,7 @@ import java.util.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -30,7 +31,6 @@ import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.lens.api.LensException;
-import org.apache.lens.cube.metadata.CubeMetastoreClient;
import org.apache.lens.cube.metadata.MetastoreConstants;
import org.apache.lens.cube.parse.HQLParser;
import org.apache.lens.driver.jdbc.ColumnarSQLRewriter;
@@ -593,8 +593,11 @@ public class TestColumnarSQLRewriter {
SessionState.start(conf);
// Create test table
- createTable("default", "mytable", "testDB", "testTable_1");
- createTable("default", "mytable_2", "testDB", "testTable_2");
+ Database database = new Database();
+ database.setName("mydb");
+ Hive.get().createDatabase(database);
+ createTable("mydb", "mytable", "testDB", "testTable_1");
+ createTable("mydb", "mytable_2", "testDB", "testTable_2");
createTable("default", "mytable_3", "testDB", "testTable_3");
String query = "SELECT * FROM mydb.mytable t1 JOIN mydb.mytable_2 t2 ON t1.t2id = t2.id "
@@ -609,7 +612,6 @@ public class TestColumnarSQLRewriter {
System.out.println(joinTreeBeforeRewrite);
// Rewrite
- CubeMetastoreClient client = CubeMetastoreClient.getInstance(conf);
rewriter.replaceWithUnderlyingStorage(rewriter.fromAST);
String joinTreeAfterRewrite = HQLParser.getString(rewriter.fromAST);
System.out.println(joinTreeAfterRewrite);
@@ -628,7 +630,7 @@ public class TestColumnarSQLRewriter {
&& joinTreeAfterRewrite.contains("testtable_3"));
// Rewrite one more query where table and db name is not set
- createTable("default", "mytable_4", null, null);
+ createTable("mydb", "mytable_4", null, null);
String query2 = "SELECT * FROM mydb.mytable_4 WHERE a = 100";
rewriter = new ColumnarSQLRewriter();
rewriter.ast = HQLParser.parseHQL(query2);
@@ -647,8 +649,11 @@ public class TestColumnarSQLRewriter {
assertEquals(joinTreeAfterRewrite, joinTreeBeforeRewrite);
// Test a query with default db
- Hive.get().dropTable("default", "mytable");
- createTable("default", "mytable", "default", null);
+ Hive.get().dropTable("mydb", "mytable");
+ database = new Database();
+ database.setName("examples");
+ Hive.get().createDatabase(database);
+ createTable("examples", "mytable", "default", null);
String defaultQuery = "SELECT * FROM examples.mytable t1 WHERE A = 100";
rewriter = new ColumnarSQLRewriter();
@@ -662,10 +667,10 @@ public class TestColumnarSQLRewriter {
assertFalse(joinTreeAfterRewrite.contains("examples"), joinTreeAfterRewrite);
System.out.println("default case: " + joinTreeAfterRewrite);
- Hive.get().dropTable("default", "mytable");
- Hive.get().dropTable("default", "mytable_2");
+ Hive.get().dropTable("examples", "mytable");
+ Hive.get().dropTable("mydb", "mytable_2");
Hive.get().dropTable("default", "mytable_3");
- Hive.get().dropTable("default", "mytable_4");
+ Hive.get().dropTable("mydb", "mytable_4");
}
/**
[46/50] incubator-lens git commit: Added javadoc and apt files.
Posted by am...@apache.org.
Added javadoc and apt files.
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/403cb7d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/403cb7d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/403cb7d0
Branch: refs/heads/toapache
Commit: 403cb7d07f6aa6be88a51d1990ff32f0122c1a8e
Parents: 6e64562
Author: Rajat Khandelwal <ra...@inmobi.com>
Authored: Thu Nov 13 12:17:03 2014 +0530
Committer: Rajat Khandelwal <ra...@inmobi.com>
Committed: Thu Nov 13 16:33:27 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 2 +-
.../DurationBasedQueryPriorityDecider.java | 86 +++++++++++++++++---
.../apache/lens/driver/hive/TestHiveDriver.java | 6 ++
.../lens/server/api/driver/MockDriver.java | 7 --
.../api/priority/CostToPriorityRangeConf.java | 27 ++++++
.../api/priority/QueryPriorityDecider.java | 9 +-
.../lens/server/api/priority/RangeConf.java | 56 +++++++++++--
.../server/api/query/AbstractQueryContext.java | 1 +
.../api/query/DriverSelectorQueryContext.java | 18 +++-
.../server/api/query/ExplainQueryContext.java | 8 ++
.../server/query/QueryExecutionServiceImpl.java | 1 -
.../lens/server/query/TestQueryService.java | 6 +-
src/site/apt/admin/config.apt | 2 +-
src/site/apt/admin/hivedriver-config.apt | 12 ++-
14 files changed, 205 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 89ca7d1..65d6988 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -77,7 +77,7 @@ public class HiveDriver implements LensDriver {
public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
/** The Constant HS2_CONNECTION_EXPIRY_DELAY. */
- public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hive.connection.expiry.delay";
+ public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hive.hs2.connection.expiry.delay";
public static final String HS2_CALCULATE_PRIORITY = "lens.driver.hive.calculate.priority";
public static final String HS2_PARTITION_WEIGHT_MONTHLY = "lens.driver.hive.priority.partition.weight.monthly";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
index 5965309..c9ab9bd 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.lens.driver.hive.priority;
import org.apache.lens.api.LensException;
@@ -12,24 +30,59 @@ import java.util.List;
import java.util.Map;
public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
+
+ /** Partition Weights for priority calculation based on selected partitions **/
+
+ /** weight of monthly partition **/
private float monthlyPartitionWeight;
+ /** weight of daily partition **/
private float dailyPartitionWeight;
+ /** weight of hourly partition **/
private float hourlyPartitionWeight;
- public DurationBasedQueryPriorityDecider(float mw, float dw, float hw){
- monthlyPartitionWeight = mw;
- dailyPartitionWeight = dw;
- hourlyPartitionWeight = hw;
+ /**
+ * Constructor. Takes three weights for partitions.
+ * @param mmonthlyPartitoinWeight
+ * @param dailyPartitionWeight
+ * @param hourlyPartitionWeight
+ */
+ public DurationBasedQueryPriorityDecider(float mmonthlyPartitoinWeight,
+ float dailyPartitionWeight, float hourlyPartitionWeight){
+ monthlyPartitionWeight = mmonthlyPartitoinWeight;
+ this.dailyPartitionWeight = dailyPartitionWeight;
+ this.hourlyPartitionWeight = hourlyPartitionWeight;
}
+
+ /**
+ * Hard Coded
+ * Arbitrary for now. Will need to tune it.
+ * Some perspective wrt default weights(1 for hourly, 0.75 for daily, 0.5 for monthly)
+ * For exclusively hourly data this translates to VERY_HIGH,7days,HIGH,30days,NORMAL,90days,LOW
+ * FOR exclusively daily data this translates to VERY_HIGH,9days,HIGH,40days,NORMAL,120days,LOW
+ * for exclusively monthly data this translates to VERY_HIGH,never,HIGH,1month,NORMAL,6months,LOW
+ */
static final CostToPriorityRangeConf costToPriorityRangeMap =
- // Hard Coded
- // Arbitrary for now. Will need to tune it.
+
new CostToPriorityRangeConf("VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW");
- public Priority decidePriority(AbstractQueryContext queryContext) throws LensException {
- return costToPriorityRangeMap.get(getDurationCost(queryContext));
+ /**
+ * The Implementation
+ * @param abstractQueryContext
+ * @return decided Priority
+ * @throws LensException
+ * Exception occurs mostly when one of drivers/explained queries/plans is null
+ */
+ public Priority decidePriority(AbstractQueryContext abstractQueryContext) throws LensException {
+ return costToPriorityRangeMap.get(getDurationCost(abstractQueryContext));
}
+ /**
+ * Extract partitions from AbstractQueryContext. Hive currently gives partitions in the format
+ * {a:[dt partition1, dt partition2]...}. This method removes the "dt"
+ * @param queryContext
+ * @return all the tables along with their selected partitions.
+ * @throws LensException
+ */
protected Map<String,List<String>> extractPartitions(AbstractQueryContext queryContext) throws LensException{
Map<String, List<String>> partitions = new HashMap<String, List<String>>();
for(Map.Entry<String, List<String>> entry: queryContext.getSelectedDriverQueryPlan().getPartitions().entrySet()) {
@@ -42,6 +95,12 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
return partitions;
}
+ /**
+ * Calculates total cost based on weights of selected tables and their selected partitions
+ * @param queryContext
+ * @return Query Cost
+ * @throws LensException
+ */
float getDurationCost(AbstractQueryContext queryContext) throws LensException {
final Map<String, List<String>> partitions = extractPartitions(queryContext);
@@ -49,14 +108,21 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
for(String table: partitions.keySet()) {
for(String partition: partitions.get(table)) {
if(!partition.equals("latest")) {
- cost += queryContext.getSelectedDriverQueryPlan().getTableWeight(table) * getPartitionCost(partition);
+ cost +=
+ queryContext.getSelectedDriverQueryPlan().getTableWeight(table) * getNormalizedPartitionCost(partition);
}
}
}
return cost;
}
- float getPartitionCost(String partition) throws LensException {
+ /**
+ * Normalized cost of a partition. PartitionWeight multiplied by number of days in that partition.
+ * @param partition
+ * @return normalized cost.
+ * @throws LensException
+ */
+ float getNormalizedPartitionCost(String partition) throws LensException {
switch (partition.length()) {
case 7: //monthly
return 30 * monthlyPartitionWeight;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 403fbfb..dfbdb79 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -768,6 +768,12 @@ public class TestHiveDriver {
assertEquals(ctx.getHdfsoutPath(), path);
driver.closeQuery(plan2.getHandle());
}
+
+ /**
+ * Testing Duration Based Priority Logic by mocking everything except partitions.
+ * @throws IOException
+ * @throws LensException
+ */
@Test
public void testPriority() throws IOException, LensException {
final MockDriver mockDriver = new MockDriver();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockDriver.java
index 8487058..6763e2e 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -101,13 +101,6 @@ public class MockDriver implements LensDriver {
@Getter @Setter
private Map<String, List<String>> partitions;
-// public Map<String, List<String>> getPartitions() {
-// return partitions;
-// }
-// public void setPartitions(Map<String, List<String>> partitions) {
-// this.partitions = partitions;
-// }
-
@Override
public String getPlan() {
return query;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
index 995666a..1310244 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
@@ -23,20 +23,47 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.lens.api.Priority;
+/**
+ * Cost to priority range conf class.
+ * @see RangeConf
+ */
+
public class CostToPriorityRangeConf extends RangeConf<Float, Priority>{
+ /**
+ * Super constructor
+ * @see RangeConf#RangeConf(String)
+ * @param confValue
+ */
public CostToPriorityRangeConf(String confValue) {
super(confValue);
}
+
+ /**
+ * Parse key method
+ * @param s
+ * @return parsed float from string s
+ * @see RangeConf#parseKey(String)
+ */
@Override
protected Float parseKey(String s) {
return Float.parseFloat(s);
}
+ /**
+ * Parse value method
+ * @param s
+ * @return parsed Priority from String s
+ * @see RangeConf#parseValue(String)
+ */
@Override
protected Priority parseValue(String s) {
return Priority.valueOf(s);
}
+ /**
+ * Default value is "NORMAL". Which implies {@link #get(Comparable)} will always return Priotity.NORMAL
+ * @return "NORMAL"
+ */
@Override
protected String getDefaultConf() {
return Priority.NORMAL.toString();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
index 5fb0bda..9a3b2f2 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
@@ -24,7 +24,14 @@ import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.log4j.Logger;
public interface QueryPriorityDecider {
-
+ /** The Constant LOG. */
public static final Logger LOG = Logger.getLogger(QueryPriorityDecider.class);
+
+ /**
+ *
+ * @param queryContext
+ * @return calculated Priority based on the explained plans for each driver
+ * @throws LensException when can't decide priority.
+ */
public Priority decidePriority(AbstractQueryContext queryContext) throws LensException;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
index c999fee..71303f8 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
@@ -24,9 +24,42 @@ import org.apache.lens.server.api.LensConfConstants;
import java.util.TreeMap;
+/**
+ * Class for storing range configurations. An Example would be grading system.
+ * The value F,30,D,40,C,60,B,80,A corresponds to a system where
+ * - inf < marks <= 30 : F
+ * 30 < marks <= 40 : D
+ * 40 < marks <= 60 : C
+ * 60 < marks <= 80 : B
+ * 80 < marks <= + Inf : A
+ *
+ * rangeConfInstance.get(marks) would give you the grade depending on the range.
+ *
+ * The utility is for easily storing range configs in config xml files.
+ *
+ * Implementation is done by storing the least value(floor) and keeping a treemap on rest of values
+ *
+ * @param <K> Key type. Integer in the grade range example
+ * @param <V> Value Type. String(or a Grade class) in the grade range example
+ */
+
public abstract class RangeConf<K extends Comparable<K>, V> {
+
+ /**
+ * The data structure is one map and one floor value
+ *
+ */
+
+ /** Treemap of lower boundary value and next grade */
TreeMap<K, V> map = new TreeMap<K, V>();
+
+ /** Lowest boundary value. */
V floor;
+
+ /**
+ * Constructor. parses the string to form the data structure.
+ * @param confValue
+ */
RangeConf(String confValue) {
if(confValue == null || confValue.isEmpty()) {
confValue = getDefaultConf();
@@ -38,22 +71,27 @@ public abstract class RangeConf<K extends Comparable<K>, V> {
map.put(parseKey(split[i]), parseValue(split[i + 1]));
}
}
-
+ /** parse key type from its string representation */
protected abstract K parseKey(String s);
+ /** parse value type from its string representation */
protected abstract V parseValue(String s);
+
+ /** When null/blank conf string passed, this would be the value from which data structure will be formed */
protected abstract String getDefaultConf();
+
+ /**
+ * Get method.
+ * @param key
+ * @return the value depending on which range the key lies in
+ */
public V get(K key) {
return map.floorEntry(key) == null ? floor : map.floorEntry(key).getValue();
}
- static String getFirstNonNullValueFromConf(Configuration conf, String... keys) {
- for (String key: keys){
- if(conf.get(key) != null) {
- return conf.get(key);
- }
- }
- return null;
- }
+ /**
+ * toString representation
+ * @return string representation
+ */
@Override
public String toString() {
return "lower value: " + floor + ", map: " + map;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 72a893c..0b3e976 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -26,6 +26,7 @@ import org.apache.lens.api.LensConf;
import java.io.Serializable;
public abstract class AbstractQueryContext extends DriverSelectorQueryContext implements Serializable {
+ /** The Constant LOG */
public static final Log LOG = LogFactory.getLog(AbstractQueryContext.class);
/** The user query. */
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 6847162..0098a1a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -31,6 +31,8 @@ import java.util.HashMap;
import java.util.Map;
public abstract class DriverSelectorQueryContext {
+
+ /** The constant LOG */
public static final Log LOG = LogFactory.getLog(DriverSelectorQueryContext.class);
/** The conf. */
@@ -46,15 +48,24 @@ public abstract class DriverSelectorQueryContext {
@Getter
@Setter
protected String driverQuery;
-
+ /** Map of driver to driver query */
@Getter
protected Map<LensDriver, String> driverQueries;
+ /** Map of driver to query plan */
@Getter
protected Map<LensDriver, DriverQueryPlan> driverQueryPlans;
+ /** Map of exceptions occurred while trying to generate plans by explain call */
protected Map<LensDriver, Exception> driverQueryPlanGenerationErrors;
+ /**
+ * Sets driver queries, generates plans for each driver by calling explain with respective queries,
+ * Sets driverQueryPlans
+ * @param driverQueries
+ * @throws LensException
+ * @see #driverQueryPlans
+ */
public void setDriverQueriesAndPlans(Map<LensDriver, String> driverQueries) throws LensException {
driverQueryPlanGenerationErrors = new HashMap<LensDriver, Exception>();
this.driverQueries = driverQueries;
@@ -71,6 +82,11 @@ public abstract class DriverSelectorQueryContext {
}
}
+ /**
+ * Return selected driver's query plan, but check for null conditions first.
+ * @return DriverQueryPlan of Selected Driver
+ * @throws LensException
+ */
public DriverQueryPlan getSelectedDriverQueryPlan() throws LensException {
if(getDriverQueryPlans() == null) {
throw new LensException("No Driver query plans. Check if re-write happened or not");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
index 86fd727..e248437 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
@@ -20,7 +20,15 @@ package org.apache.lens.server.api.query;
import org.apache.hadoop.conf.Configuration;
+/**
+ * The class ExplainQueryContext
+ */
public class ExplainQueryContext extends AbstractQueryContext{
+ /**
+ * Constructor. Only needs user query and conf.
+ * @param query
+ * @param qconf
+ */
public ExplainQueryContext(String query, Configuration qconf) {
userQuery = query;
conf = qconf;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index c2c5810..a3144f0 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -919,7 +919,6 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
LensDriver driver = driverSelector.select(ctx, conf);
ctx.setSelectedDriver(driver);
- //TODO: remove this. Note the Redundancy.
ctx.setDriverQuery(ctx.getDriverQueries().get(ctx.getSelectedDriver()));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index fc10723..8334d95 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -344,9 +344,9 @@ public class TestQueryService extends LensJerseyTest {
Thread.sleep(1000);
}
assertTrue(ctx.getSubmissionTime() > 0);
-// assertTrue(ctx.getLaunchTime() > 0);
-// assertTrue(ctx.getDriverStartTime() > 0);
-// assertTrue(ctx.getDriverFinishTime() > 0);
+ assertTrue(ctx.getLaunchTime() > 0);
+ assertTrue(ctx.getDriverStartTime() > 0);
+ assertTrue(ctx.getDriverFinishTime() > 0);
assertTrue(ctx.getFinishTime() > 0);
Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index abaf2dc..e982b07 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -174,7 +174,7 @@ Lens server configuration
*--+--+---+--+
|75|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|76|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. example schema of the file is: user1, user2, user3 = clusteruser1 user4, user5 = clusteruser2 * = clusteruser3 |
+|76|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default |
*--+--+---+--+
|77|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
*--+--+---+--+
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/403cb7d0/src/site/apt/admin/hivedriver-config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/hivedriver-config.apt b/src/site/apt/admin/hivedriver-config.apt
index 30b53bc..f59faab 100644
--- a/src/site/apt/admin/hivedriver-config.apt
+++ b/src/site/apt/admin/hivedriver-config.apt
@@ -44,8 +44,16 @@ Hive driver configuration
*--+--+---+--+
|10|lens.cube.query.replace.timedim|true|Tells whether timedim attribute queried in the time range should be replaced with its corresponding partition column name.|
*--+--+---+--+
-|11|lens.driver.hive.connection.class|org.apache.lens.driver.hive.EmbeddedThriftConnection|The connection class from HiveDriver to HiveServer. The default is an embedded connection which does not require a remote hive server. For connecting to a hiveserver end point, remote connection should be used. The possible values are org.apache.lens.driver.hive.EmbeddedThriftConnection and org.apache.lens.driver.hive.RemoteThriftConnection. |
+|11|lens.driver.hive.calculate.priority|true|Whether priority should be calculated for hive mr jobs or not|
*--+--+---+--+
-|12|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2|
+|12|lens.driver.hive.connection.class|org.apache.lens.driver.hive.EmbeddedThriftConnection|The connection class from HiveDriver to HiveServer. The default is an embedded connection which does not require a remote hive server. For connecting to a hiveserver end point, remote connection should be used. The possible values are org.apache.lens.driver.hive.EmbeddedThriftConnection and org.apache.lens.driver.hive.RemoteThriftConnection. |
+*--+--+---+--+
+|13|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2|
+*--+--+---+--+
+|14|lens.driver.hive.priority.partition.weight.daily|0.75|Weight of monthly partition in cost calculation|
+*--+--+---+--+
+|15|lens.driver.hive.priority.partition.weight.hourly|1.0|Weight of monthly partition in cost calculation|
+*--+--+---+--+
+|16|lens.driver.hive.priority.partition.weight.monthly|0.5|Weight of monthly partition in cost calculation|
*--+--+---+--+
The configuration parameters and their default values
[04/50] incubator-lens git commit: Make hive tests ot run under suite
Posted by am...@apache.org.
Make hive tests ot run under suite
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/09673fec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/09673fec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/09673fec
Branch: refs/heads/toapache
Commit: 09673fec77e9184d9ff7a4cd7b008d97d27ccd76
Parents: c0a92e3
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 6 17:49:27 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 6 17:49:27 2014 +0530
----------------------------------------------------------------------
lens-driver-hive/pom.xml | 3 +++
lens-driver-hive/testng.xml | 8 ++++++--
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/09673fec/lens-driver-hive/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml
index 6ec3183..b17cf96 100644
--- a/lens-driver-hive/pom.xml
+++ b/lens-driver-hive/pom.xml
@@ -50,6 +50,9 @@
<environmentVariables>
<MVN_CLASSPATH_FILE>${mvn.classpath.file}</MVN_CLASSPATH_FILE>
</environmentVariables>
+ <suiteXmlFiles>
+ <suiteXmlFile>testng.xml</suiteXmlFile>
+ </suiteXmlFiles>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/09673fec/lens-driver-hive/testng.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/testng.xml b/lens-driver-hive/testng.xml
index 43e73d3..b98d252 100644
--- a/lens-driver-hive/testng.xml
+++ b/lens-driver-hive/testng.xml
@@ -22,11 +22,15 @@
<suite name="lens-driver-hive" verbose="1" >
+ <test name="TestHiveDriver">
+ <classes>
+ <class name="org.apache.lens.driver.hive.TestHiveDriver"/>
+ </classes>
+ </test>
+
<test name="TestRemoteHiveDriver">
<classes>
<class name="org.apache.lens.driver.hive.TestRemoteHiveDriver"/>
</classes>
</test>
-
-
</suite>
[22/50] incubator-lens git commit: Merge pull request #172 from
prongs/merge_db
Posted by am...@apache.org.
Merge pull request #172 from prongs/merge_db
redundant confs removed
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/fab4c569
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/fab4c569
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/fab4c569
Branch: refs/heads/toapache
Commit: fab4c569f04e793a44aeceadee52a86b40352ddd
Parents: 4ffc235 a23f065
Author: Amareshwari <am...@inmobi.com>
Authored: Tue Nov 11 09:35:47 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Tue Nov 11 09:35:47 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/server/api/LensConfConstants.java | 12 ------------
1 file changed, 12 deletions(-)
----------------------------------------------------------------------
[38/50] incubator-lens git commit: Fix Hive client conf in
ColumnarSQLRewriter
Posted by am...@apache.org.
Fix Hive client conf in ColumnarSQLRewriter
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/af8dfc40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/af8dfc40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/af8dfc40
Branch: refs/heads/toapache
Commit: af8dfc40f98c73ecfe061b8c9ddea86472df4707
Parents: e21f750
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 13 12:34:36 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 13 12:34:36 2014 +0530
----------------------------------------------------------------------
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 24 +++++---------------
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 9 ++++++--
.../apache/lens/driver/jdbc/QueryRewriter.java | 7 ++++++
3 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/af8dfc40/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index 6110895..7961b39 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -45,7 +45,7 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
public class ColumnarSQLRewriter implements QueryRewriter {
/** The conf. */
- private Configuration conf;
+ private HiveConf conf;
/** The clause name. */
private String clauseName = null;
@@ -151,23 +151,12 @@ public class ColumnarSQLRewriter implements QueryRewriter {
/**
* Instantiates a new columnar sql rewriter.
- *
- * @param conf
- * the conf
- * @param query
- * the query
*/
- public ColumnarSQLRewriter(HiveConf conf, String query) {
- // super(conf);
- this.conf = conf;
- this.query = query;
+ public ColumnarSQLRewriter() {
}
- /**
- * Instantiates a new columnar sql rewriter.
- */
- public ColumnarSQLRewriter() {
- // super(conf);
+ public void setConf(Configuration conf) {
+ this.conf = new HiveConf(conf, ColumnarSQLRewriter.class);
}
public String getClause() {
@@ -794,7 +783,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
@Override
public synchronized String rewrite(Configuration conf, String query) throws LensException {
this.query = query;
- this.conf = conf;
StringBuilder mergedQuery = new StringBuilder();
rewrittenQuery.setLength(0);
String queryReplacedUdf = "";
@@ -909,7 +897,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* the hive exception
*/
String getUnderlyingDBName(String table) throws HiveException {
- Table tbl = Hive.get().getTable(table);
+ Table tbl = Hive.get(this.conf).getTable(table);
return tbl == null ? null : tbl.getProperty(LensConfConstants.NATIVE_DB_NAME);
}
@@ -925,7 +913,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* the hive exception
*/
String getUnderlyingTableName(String table) throws HiveException {
- Table tbl = Hive.get().getTable(table);
+ Table tbl = Hive.get(this.conf).getTable(table);
return tbl == null ? null : tbl.getProperty(LensConfConstants.NATIVE_TABLE_NAME);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/af8dfc40/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 585de38..6e79a01 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -352,6 +352,10 @@ public class JDBCDriver implements LensDriver {
public String rewrite(Configuration conf, String query) throws LensException {
return query;
}
+
+ @Override
+ public void setConf(Configuration conf) {
+ }
}
/**
@@ -441,7 +445,7 @@ public class JDBCDriver implements LensDriver {
* @throws LensException
* the lens exception
*/
- protected synchronized QueryRewriter getQueryRewriter(Configuration conf) throws LensException {
+ protected synchronized QueryRewriter getQueryRewriter() throws LensException {
QueryRewriter rewriter;
Class<? extends QueryRewriter> queryRewriterClass = conf.getClass(JDBC_QUERY_REWRITER_CLASS,
DummyQueryRewriter.class, QueryRewriter.class);
@@ -455,6 +459,7 @@ public class JDBCDriver implements LensDriver {
LOG.error("Unable to create rewriter object", e);
throw new LensException(e);
}
+ rewriter.setConf(conf);
rewriterCache.put(queryRewriterClass, rewriter);
}
return rewriter;
@@ -505,7 +510,7 @@ public class JDBCDriver implements LensDriver {
throw new LensException(e);
}
- QueryRewriter rewriter = getQueryRewriter(conf);
+ QueryRewriter rewriter = getQueryRewriter();
String rewrittenQuery = rewriter.rewrite(conf, query);
if (LOG.isDebugEnabled()) {
LOG.debug("Query: " + query + " rewrittenQuery: " + rewrittenQuery);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/af8dfc40/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/QueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/QueryRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/QueryRewriter.java
index dbe0e80..1c319ab 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/QueryRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/QueryRewriter.java
@@ -38,4 +38,11 @@ public interface QueryRewriter {
* the lens exception
*/
public String rewrite(Configuration conf, String query) throws LensException;
+
+ /**
+ * Set conf for the rewriter
+ *
+ * @param conf
+ */
+ public void setConf(Configuration conf);
}
[40/50] incubator-lens git commit: Merge pull request #176 from
InMobi/db-issue
Posted by am...@apache.org.
Merge pull request #176 from InMobi/db-issue
Fix Hive client conf in ColumnarSQLRewriter
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/9d3062c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/9d3062c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/9d3062c8
Branch: refs/heads/toapache
Commit: 9d3062c8a850d5207154b00df5875d5ee615c05e
Parents: 654b87f af8dfc4
Author: Amareshwari <am...@inmobi.com>
Authored: Thu Nov 13 12:57:07 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Thu Nov 13 12:57:07 2014 +0530
----------------------------------------------------------------------
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 24 +++++---------------
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 9 ++++++--
.../apache/lens/driver/jdbc/QueryRewriter.java | 7 ++++++
3 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
[47/50] incubator-lens git commit: Merge pull request #162 from
prongs/job_priority
Posted by am...@apache.org.
Merge pull request #162 from prongs/job_priority
set mapred job priority by hive driver
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/fc1c88a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/fc1c88a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/fc1c88a7
Branch: refs/heads/toapache
Commit: fc1c88a7d0dc761df4ccefb40447cdf5a2651bce
Parents: 5d15a2b 403cb7d
Author: Amareshwari <am...@inmobi.com>
Authored: Thu Nov 13 16:35:03 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Thu Nov 13 16:35:03 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/cube/MockDriver.java | 345 ------------------
.../lens/driver/cube/TestMinCostSelector.java | 33 +-
.../apache/lens/driver/cube/TestRewriting.java | 3 +-
lens-cube/src/test/resources/hive-site.xml | 2 +-
.../org/apache/lens/driver/hive/HiveDriver.java | 38 +-
.../DurationBasedQueryPriorityDecider.java | 137 +++++++
.../src/main/resources/hivedriver-default.xml | 26 +-
.../apache/lens/driver/hive/TestHiveDriver.java | 42 ++-
.../lens/driver/hive/TestRemoteHiveDriver.java | 5 +-
.../src/test/resources/priority_tests.txt | 4 +
.../lens/server/api/driver/DriverSelector.java | 9 +-
.../server/api/driver/MinQueryCostSelector.java | 53 +--
.../lens/server/api/driver/MockDriver.java | 354 +++++++++++++++++++
.../lens/server/api/driver/MockFailDriver.java | 37 ++
.../server/api/driver/MockQueryContext.java | 30 ++
.../api/priority/CostToPriorityRangeConf.java | 71 ++++
.../api/priority/QueryPriorityDecider.java | 37 ++
.../lens/server/api/priority/RangeConf.java | 99 ++++++
.../server/api/query/AbstractQueryContext.java | 39 ++
.../api/query/DriverSelectorQueryContext.java | 103 ++++++
.../server/api/query/ExplainQueryContext.java | 36 ++
.../server/api/query/PreparedQueryContext.java | 25 +-
.../lens/server/api/query/QueryContext.java | 26 +-
.../server/query/QueryExecutionServiceImpl.java | 38 +-
.../lens/server/query/TestQueryService.java | 10 +-
src/site/apt/admin/hivedriver-config.apt | 12 +-
26 files changed, 1100 insertions(+), 514 deletions(-)
----------------------------------------------------------------------
[14/50] incubator-lens git commit: Merge pull request #166 from
prongs/remove_cubedriver
Posted by am...@apache.org.
Merge pull request #166 from prongs/remove_cubedriver
Remove cubedriver
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/75ce83f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/75ce83f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/75ce83f6
Branch: refs/heads/toapache
Commit: 75ce83f682955c1cc4b51131f474159e48fa8d87
Parents: 01a5c35 dbe8143
Author: Amareshwari <am...@inmobi.com>
Authored: Fri Nov 7 14:37:32 2014 +0530
Committer: Amareshwari <am...@inmobi.com>
Committed: Fri Nov 7 14:37:32 2014 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/cube/CubeDriver.java | 604 -------------------
.../apache/lens/driver/cube/RewriteUtil.java | 26 +-
.../apache/lens/driver/cube/TestCubeDriver.java | 174 ------
.../lens/driver/cube/TestMinCostSelector.java | 2 +-
.../server/api/driver/MinQueryCostSelector.java | 83 +++
.../server/query/QueryExecutionServiceImpl.java | 3 +-
6 files changed, 99 insertions(+), 793 deletions(-)
----------------------------------------------------------------------
[49/50] incubator-lens git commit: Remove the usage of inmobi and
grill
Posted by am...@apache.org.
Remove the usage of inmobi and grill
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/a47ecd5f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/a47ecd5f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/a47ecd5f
Branch: refs/heads/toapache
Commit: a47ecd5f29013535487fc21ca7d424b436a1f8ae
Parents: ea0bfc2
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Thu Nov 13 17:04:50 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Thu Nov 13 17:04:50 2014 +0530
----------------------------------------------------------------------
README.md | 2 +-
lens-api/pom.xml | 2 +-
lens-cli/pom.xml | 2 +-
lens-client-dist/pom.xml | 2 +-
lens-client/pom.xml | 2 +-
lens-cube/pom.xml | 2 +-
.../apache/lens/cube/parse/TestTimeRangeExtractor.java | 1 -
lens-dist/pom.xml | 2 +-
lens-driver-hive/pom.xml | 2 +-
lens-driver-impala/pom.xml | 2 +-
lens-driver-jdbc/pom.xml | 2 +-
lens-examples/pom.xml | 2 +-
lens-ml-lib/pom.xml | 2 +-
.../main/java/org/apache/lens/api/ml/TestReport.java | 2 +-
lens-query-lib/pom.xml | 2 +-
lens-server-api/pom.xml | 2 +-
lens-server/enunciate.xml | 3 +--
lens-server/pom.xml | 2 +-
lens-storage-db/pom.xml | 2 +-
pom.xml | 2 +-
src/site/apt/admin/config-server.apt | 5 +++--
src/site/apt/developer/contribute.apt | 2 +-
src/site/apt/gettingstarted/install-steps.apt | 2 +-
src/site/apt/user/jdbc-client.apt | 2 +-
src/site/site.xml | 12 +++++-------
tools/scripts/generate-site-public.sh | 2 +-
26 files changed, 31 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 1a92565..07a3249 100644
--- a/README.md
+++ b/README.md
@@ -3,4 +3,4 @@ Lens
Lens is a unified Analytics Platform
-[Detailed documentation for the project is available here](https://inmobi.github.io/grill)
+[Detailed documentation for the project is available here](https://lens.incubator.apache.org)
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-api/pom.xml b/lens-api/pom.xml
index 34e3865..f685f1b 100644
--- a/lens-api/pom.xml
+++ b/lens-api/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-cli/pom.xml
----------------------------------------------------------------------
diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml
index 2f74a93..03b12ab 100644
--- a/lens-cli/pom.xml
+++ b/lens-cli/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-client-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-client-dist/pom.xml b/lens-client-dist/pom.xml
index d821229..1860c83 100644
--- a/lens-client-dist/pom.xml
+++ b/lens-client-dist/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-client-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-client/pom.xml
----------------------------------------------------------------------
diff --git a/lens-client/pom.xml b/lens-client/pom.xml
index 97c56e7..fad03fc 100644
--- a/lens-client/pom.xml
+++ b/lens-client/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-cube/pom.xml
----------------------------------------------------------------------
diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml
index a1e6be5..2a461d1 100644
--- a/lens-cube/pom.xml
+++ b/lens-cube/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-cube</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeExtractor.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeExtractor.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeExtractor.java
index f198249..f68715f 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeExtractor.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeExtractor.java
@@ -75,7 +75,6 @@ public class TestTimeRangeExtractor extends TestQueryRewrite {
@Test
public void testNoNPE() throws Exception {
- // GRILL-38 NPE in extracting time range
String timeRange = " time_range_in(dt, '" + dateTwoDaysBack + "','" + dateNow + "')";
String q1 = "SELECT cityid, testCube.msr2 from testCube where " + timeRange + " AND cityid IS NULL";
rewrite(driver, q1);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml
index 987839d..4ea53fe 100644
--- a/lens-dist/pom.xml
+++ b/lens-dist/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-driver-hive/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml
index 5d300c2..f030251 100644
--- a/lens-driver-hive/pom.xml
+++ b/lens-driver-hive/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-driver-impala/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-impala/pom.xml b/lens-driver-impala/pom.xml
index 21f56f5..aaeb08b 100644
--- a/lens-driver-impala/pom.xml
+++ b/lens-driver-impala/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-driver-impala</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-driver-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml
index 44f43d6..c0d827d 100644
--- a/lens-driver-jdbc/pom.xml
+++ b/lens-driver-jdbc/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-driver-jdbc</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-examples/pom.xml
----------------------------------------------------------------------
diff --git a/lens-examples/pom.xml b/lens-examples/pom.xml
index daaa04d..c3362c1 100644
--- a/lens-examples/pom.xml
+++ b/lens-examples/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-ml-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml
index 028b3cd..f9e7e0f 100644
--- a/lens-ml-lib/pom.xml
+++ b/lens-ml-lib/pom.xml
@@ -27,7 +27,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-ml-lib/src/main/java/org/apache/lens/api/ml/TestReport.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/api/ml/TestReport.java b/lens-ml-lib/src/main/java/org/apache/lens/api/ml/TestReport.java
index 90a71b5..4c0bf51 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/api/ml/TestReport.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/api/ml/TestReport.java
@@ -115,7 +115,7 @@ public class TestReport {
builder.append("Algorithm: ").append(algorithm).append('\n');
builder.append("Report id: ").append(reportID).append('\n');
builder.append("Model id: ").append(modelID).append('\n');
- builder.append("Grill Query id: ").append(queryID).append('\n');
+ builder.append("Lens Query id: ").append(queryID).append('\n');
builder.append("Feature columns: ").append(featureColumns).append('\n');
builder.append("Labelled column: ").append(labelColumn).append('\n');
builder.append("Predicted column: ").append(outputColumn).append('\n');
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-query-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-query-lib/pom.xml b/lens-query-lib/pom.xml
index 86ab4cd..3ca9c3c 100644
--- a/lens-query-lib/pom.xml
+++ b/lens-query-lib/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-query-lib</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-server-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server-api/pom.xml b/lens-server-api/pom.xml
index f03cfd6..5ff2837 100644
--- a/lens-server-api/pom.xml
+++ b/lens-server-api/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-server-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-server/enunciate.xml
----------------------------------------------------------------------
diff --git a/lens-server/enunciate.xml b/lens-server/enunciate.xml
index f727671..b4fafae 100644
--- a/lens-server/enunciate.xml
+++ b/lens-server/enunciate.xml
@@ -21,9 +21,8 @@
-->
<enunciate label="lens" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="http://enunciate.codehaus.org/schemas/enunciate-1.25.xsd">
- <deployment protocol="https" host="github.com" context="/pages/InMobi/lens/wsdocs"/>
<namespaces>
- <namespace uri="http://github.com/InMobi/grill" id="grill"/>
+ <namespace uri="https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git" id="lens"/>
</namespaces>
<services>
<rest defaultRestSubcontext="/">
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-server/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index 42192d9..9a8fb7d 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/lens-storage-db/pom.xml
----------------------------------------------------------------------
diff --git a/lens-storage-db/pom.xml b/lens-storage-db/pom.xml
index bc0ee3f..80a640b 100644
--- a/lens-storage-db/pom.xml
+++ b/lens-storage-db/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>lens-parent</artifactId>
<groupId>org.apache.lens</groupId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>lens-storage-db</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e8beee3..42f0df4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -47,7 +47,7 @@
<groupId>org.apache.lens</groupId>
<artifactId>lens-parent</artifactId>
- <version>1.2.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<name>Lens</name>
<packaging>pom</packaging>
<description>Unified Analytics Platform</description>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/src/site/apt/admin/config-server.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config-server.apt b/src/site/apt/admin/config-server.apt
index eb45b04..9349bfc 100644
--- a/src/site/apt/admin/config-server.apt
+++ b/src/site/apt/admin/config-server.apt
@@ -34,9 +34,10 @@ Configuring lens server
The supported drivers are configured through classnames in this release. The configuration
for each driver can be specified in their site.xml file. For example, HiveDriver's
configuration should be specified in hive-site.xml. Supported drivers in this version
- are com.inmobi.lens.driver.hive.HiveDriver and com.inmobi.lens.driver.jdbc.JDBCDriver.
+ are org.apache.lens.driver.hive.HiveDriver and org.apache.lens.driver.jdbc.JDBCDriver.
The configuration parameters and
- their detailed description and default values for HiveDriver are available {{{./hivedriver-config.html} here}}
+ their detailed description and default values for HiveDriver
+ are available {{{./hivedriver-config.html} here}}
and for JDBCDriver available {{{./jdbcdriver-config.html} here}}.
The overridden configuration for HiveDriver and JDBCDriver can be part of hivedriver-site.xml
and jdbc-driver-site.xml respectively.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/src/site/apt/developer/contribute.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/developer/contribute.apt b/src/site/apt/developer/contribute.apt
index e1c5450..dcf033c 100644
--- a/src/site/apt/developer/contribute.apt
+++ b/src/site/apt/developer/contribute.apt
@@ -29,7 +29,7 @@ Developer Documentation : How to contribute to Apache Lens?
** Source Repository
Lens uses {{{http://git-scm.com/} git}} for its code repository. The repository is available at
- {{{https://github.com/InMobi/grill} https://github.com/InMobi/grill}}.
+ {{{https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git} https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git}}.
** Build tools
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/src/site/apt/gettingstarted/install-steps.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/gettingstarted/install-steps.apt b/src/site/apt/gettingstarted/install-steps.apt
index e4d860f..9560e0c 100644
--- a/src/site/apt/gettingstarted/install-steps.apt
+++ b/src/site/apt/gettingstarted/install-steps.apt
@@ -27,7 +27,7 @@ Installing and running lens
+---+
- git clone https://github.com/InMobi/grill
+ git clone https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git
cd lens
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/src/site/apt/user/jdbc-client.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/jdbc-client.apt b/src/site/apt/user/jdbc-client.apt
index 58b418e..9ac3577 100644
--- a/src/site/apt/user/jdbc-client.apt
+++ b/src/site/apt/user/jdbc-client.apt
@@ -24,7 +24,7 @@ JDBC Client
<<Using JDBC driver.>>
- The main driver class for accessing lens server is <<<com.inmobi.lens.client.jdbc.LensJdbcDriver.java>>>
+ The main driver class for accessing lens server is <<<org.apache.lens.client.jdbc.LensJdbcDriver.java>>>
Add <<<lens-client-site.xml>>> in the class path and configure following parameters as described in {{{./client-config.html}here}}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/src/site/site.xml
----------------------------------------------------------------------
diff --git a/src/site/site.xml b/src/site/site.xml
index 5d90886..76c83ef 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -25,10 +25,9 @@
<publishDate position="right"/>
<version position="right"/>
<bannerLeft>
- <name>Lens Analytics Platform for InMobi</name>
+ <name>Lens Analytics Platform</name>
<title>Lens Analytics Platform</title>
- <src>http://inmobi.github.io/grill/grill-logo.png</src>
- <href>http://inmobi.github.io/grill</href>
+ <href>http://lens.incubator.apache.org</href>
</bannerLeft>
<body>
@@ -60,8 +59,8 @@
</menu>
<menu ref="reports"/>
<breadcrumbs>
- <item name="InMobi" href="https://github.com/InMobi" />
- <item name="Lens" href="https://github.com/InMobi/grill" />
+ <item name="ASF" href="http://www.apache.org" />
+ <item name="Lens" href="http://lens.incubator.apache.org" />
</breadcrumbs>
</body>
<skin>
@@ -70,7 +69,6 @@
<version>1.1</version>
</skin>
<poweredBy>
- <logo name="inmobi" href="http://www.inmobi.com/"
- img="../images/inmobi-logo.jpg"/>
+ <logo name="ASF" href="http://www.apache.org/"/>
</poweredBy>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/a47ecd5f/tools/scripts/generate-site-public.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/generate-site-public.sh b/tools/scripts/generate-site-public.sh
index fb09f36..afe956b 100755
--- a/tools/scripts/generate-site-public.sh
+++ b/tools/scripts/generate-site-public.sh
@@ -24,7 +24,7 @@ die() {
exit 1
}
-REPO=git@github.com:InMobi/grill.git
+REPO=https://git-wip-us.apache.org/repos/asf?p=incubator-lens.git
TMP=/tmp/lens-site-stage
STAGE=`pwd`/target/staging
REST_DIR=`pwd`/lens-server/target/site/wsdocs